From 08e36d15e922695d9856cdb561634ec831f7e1fc Mon Sep 17 00:00:00 2001 From: Abhishek Mallepalli Date: Fri, 28 Aug 2026 18:03:27 -0700 Subject: [PATCH] Vendor oneTBB 2023.0.0 (headers, import libs, DLLs) Adds the oneTBB 2023.0.0 distribution as a base layer, with no consumer wiring. Nothing in the engine references these files yet, so this commit is functionally inert: the build output is unchanged. - src/3rd party/tbb, src/3rd party/oneapi: public headers, replacing the stale TBB header drop that was already in the tree. - sdk/libraries/x64: tbb.lib, tbb12.lib, tbbmalloc.lib, tbbmalloc_proxy.lib (needed at link time by later layers, and by oneTBB's implicit MSVC auto-linkage). - sdk/binaries: the matching tbb.dll, tbb12.dll, tbbmalloc.dll, tbbmalloc_proxy.dll. Also removes src/3rd party/serial/, which was orphaned: its only referrer was the old 3rd party/tbb/parallel_for.h, which this drop replaces. .gitignore needed a fix so the import libs are actually committed. The pre-existing /**/x64/ rule excludes the *directory*, and git will not descend into an excluded directory -- so the directory exclusion is lifted first, the contents re-ignored, then the four .lib files re-included by name. --- .gitignore | 13 +- sdk/binaries/tbb.dll | Bin 391040 -> 0 bytes sdk/binaries/tbb12.dll | Bin 0 -> 350608 bytes sdk/binaries/tbbmalloc.dll | Bin 0 -> 139656 bytes sdk/binaries/tbbmalloc_proxy.dll | Bin 0 -> 85896 bytes sdk/libraries/x64/tbb.lib | Bin 102378 -> 47260 bytes sdk/libraries/x64/tbb12.lib | Bin 0 -> 47260 bytes sdk/libraries/x64/tbbmalloc.lib | Bin 0 -> 9616 bytes sdk/libraries/x64/tbbmalloc_proxy.lib | Bin 0 -> 2192 bytes src/3rd party/oneapi/tbb.h | 78 + src/3rd party/oneapi/tbb/blocked_nd_range.h | 196 + src/3rd party/oneapi/tbb/blocked_range.h | 168 + src/3rd party/oneapi/tbb/blocked_range2d.h | 111 + src/3rd party/oneapi/tbb/blocked_range3d.h | 130 + .../oneapi/tbb/cache_aligned_allocator.h | 189 + .../oneapi/tbb/collaborative_call_once.h | 257 + src/3rd party/oneapi/tbb/combinable.h | 69 + .../oneapi/tbb/concurrent_hash_map.h | 1671 +++++++ .../oneapi/tbb/concurrent_lru_cache.h | 374 ++ src/3rd party/oneapi/tbb/concurrent_map.h | 351 ++ .../oneapi/tbb/concurrent_priority_queue.h | 490 ++ src/3rd party/oneapi/tbb/concurrent_queue.h | 705 +++ src/3rd party/oneapi/tbb/concurrent_set.h | 268 + .../oneapi/tbb/concurrent_unordered_map.h | 414 ++ .../oneapi/tbb/concurrent_unordered_set.h | 333 ++ src/3rd party/oneapi/tbb/concurrent_vector.h | 1130 +++++ src/3rd party/oneapi/tbb/detail/_aggregator.h | 176 + .../oneapi/tbb/detail/_aligned_space.h | 46 + .../oneapi/tbb/detail/_allocator_traits.h | 107 + src/3rd party/oneapi/tbb/detail/_assert.h | 70 + .../tbb/detail/_attach.h} | 25 +- .../tbb/detail/_concurrent_queue_base.h | 650 +++ .../oneapi/tbb/detail/_concurrent_skip_list.h | 1352 +++++ .../tbb/detail/_concurrent_unordered_base.h | 1516 ++++++ src/3rd party/oneapi/tbb/detail/_config.h | 581 +++ .../oneapi/tbb/detail/_containers_helpers.h | 67 + src/3rd party/oneapi/tbb/detail/_exception.h | 95 + src/3rd party/oneapi/tbb/detail/_export.h | 54 + .../tbb/detail}/_flow_graph_body_impl.h | 328 +- .../tbb/detail/_flow_graph_cache_impl.h | 501 ++ .../oneapi/tbb/detail/_flow_graph_impl.h | 556 +++ .../tbb/detail/_flow_graph_indexer_impl.h | 248 + .../tbb/detail/_flow_graph_item_buffer_impl.h | 422 ++ .../oneapi/tbb/detail/_flow_graph_join_impl.h | 1544 ++++++ .../oneapi/tbb/detail/_flow_graph_node_impl.h | 891 ++++ .../tbb/detail}/_flow_graph_node_set_impl.h | 48 +- .../tbb/detail/_flow_graph_nodes_deduction.h | 300 ++ .../detail/_flow_graph_resource_limiting.h | 663 +++ .../detail/_flow_graph_tagged_buffer_impl.h | 380 ++ .../tbb/detail/_flow_graph_trace_impl.h | 365 ++ .../tbb/detail/_flow_graph_types_impl.h | 402 ++ .../oneapi/tbb/detail/_hash_compare.h | 147 + .../oneapi/tbb/detail/_intrusive_list_node.h | 41 + src/3rd party/oneapi/tbb/detail/_machine.h | 371 ++ .../oneapi/tbb/detail/_mutex_common.h | 61 + .../tbb/detail/_namespace_injection.h} | 22 +- .../oneapi/tbb/detail/_node_handle.h | 162 + .../oneapi/tbb/detail/_pipeline_filters.h | 461 ++ .../tbb/detail/_pipeline_filters_deduction.h | 46 + .../oneapi/tbb/detail/_range_common.h | 133 + src/3rd party/oneapi/tbb/detail/_rtm_mutex.h | 162 + .../oneapi/tbb/detail/_rtm_rw_mutex.h | 216 + .../oneapi/tbb/detail/_scoped_lock.h | 174 + .../oneapi/tbb/detail/_segment_table.h | 581 +++ .../oneapi/tbb/detail/_small_object_pool.h | 108 + .../tbb/detail/_string_resource.h} | 45 +- src/3rd party/oneapi/tbb/detail/_task.h | 277 ++ .../oneapi/tbb/detail/_task_handle.h | 699 +++ .../oneapi/tbb/detail/_template_helpers.h | 407 ++ src/3rd party/oneapi/tbb/detail/_utils.h | 449 ++ .../oneapi/tbb/detail/_waitable_atomic.h | 90 + .../oneapi/tbb/enumerable_thread_specific.h | 1126 +++++ src/3rd party/oneapi/tbb/flow_graph.h | 3393 +++++++++++++ .../oneapi/tbb/flow_graph_abstractions.h | 51 + src/3rd party/oneapi/tbb/global_control.h | 229 + src/3rd party/oneapi/tbb/info.h | 179 + src/3rd party/oneapi/tbb/memory_pool.h | 272 ++ src/3rd party/oneapi/tbb/mutex.h | 93 + src/3rd party/oneapi/tbb/null_mutex.h | 80 + src/3rd party/oneapi/tbb/null_rw_mutex.h | 87 + src/3rd party/oneapi/tbb/parallel_for.h | 469 ++ src/3rd party/oneapi/tbb/parallel_for_each.h | 712 +++ src/3rd party/oneapi/tbb/parallel_invoke.h | 227 + src/3rd party/oneapi/tbb/parallel_pipeline.h | 153 + src/3rd party/oneapi/tbb/parallel_reduce.h | 772 +++ src/3rd party/oneapi/tbb/parallel_scan.h | 630 +++ src/3rd party/oneapi/tbb/parallel_sort.h | 288 ++ src/3rd party/oneapi/tbb/partitioner.h | 682 +++ src/3rd party/oneapi/tbb/profiling.h | 243 + src/3rd party/oneapi/tbb/queuing_mutex.h | 193 + src/3rd party/oneapi/tbb/queuing_rw_mutex.h | 207 + src/3rd party/oneapi/tbb/rw_mutex.h | 216 + src/3rd party/oneapi/tbb/scalable_allocator.h | 335 ++ src/3rd party/oneapi/tbb/spin_mutex.h | 138 + src/3rd party/oneapi/tbb/spin_rw_mutex.h | 229 + .../tbb/task.h} | 34 +- src/3rd party/oneapi/tbb/task_arena.h | 741 +++ src/3rd party/oneapi/tbb/task_group.h | 841 ++++ .../oneapi/tbb/task_scheduler_observer.h | 116 + src/3rd party/oneapi/tbb/tbb_allocator.h | 126 + src/3rd party/oneapi/tbb/tbbmalloc_proxy.h | 65 + src/3rd party/oneapi/tbb/tick_count.h | 99 + src/3rd party/oneapi/tbb/version.h | 119 + src/3rd party/serial/tbb/parallel_for.h | 226 - src/3rd party/tbb/aggregator.h | 204 - src/3rd party/tbb/aligned_space.h | 60 - src/3rd party/tbb/atomic.h | 586 --- .../tbb_annotate.h => tbb/blocked_nd_range.h} | 19 +- src/3rd party/tbb/blocked_range.h | 155 +- src/3rd party/tbb/blocked_range2d.h | 91 +- src/3rd party/tbb/blocked_range3d.h | 110 +- src/3rd party/tbb/blocked_rangeNd.h | 150 - src/3rd party/tbb/cache_aligned_allocator.h | 196 +- src/3rd party/tbb/collaborative_call_once.h | 17 + src/3rd party/tbb/combinable.h | 75 +- src/3rd party/tbb/compat/condition_variable | 489 -- src/3rd party/tbb/compat/ppl.h | 75 - src/3rd party/tbb/compat/thread | 73 - src/3rd party/tbb/compat/tuple | 501 -- src/3rd party/tbb/concurrent_hash_map.h | 1637 +------ src/3rd party/tbb/concurrent_lru_cache.h | 277 +- src/3rd party/tbb/concurrent_map.h | 376 +- src/3rd party/tbb/concurrent_priority_queue.h | 539 +- src/3rd party/tbb/concurrent_queue.h | 466 +- src/3rd party/tbb/concurrent_set.h | 291 +- src/3rd party/tbb/concurrent_unordered_map.h | 479 +- src/3rd party/tbb/concurrent_unordered_set.h | 435 +- src/3rd party/tbb/concurrent_vector.h | 1383 +----- src/3rd party/tbb/critical_section.h | 147 - .../tbb/enumerable_thread_specific.h | 1160 +---- src/3rd party/tbb/flow_graph.h | 4335 +---------------- src/3rd party/tbb/flow_graph_abstractions.h | 40 +- src/3rd party/tbb/flow_graph_opencl_node.h | 1488 ------ src/3rd party/tbb/global_control.h | 65 +- src/3rd party/tbb/index.html | 29 - src/3rd party/tbb/info.h | 39 +- src/3rd party/tbb/internal/_aggregator_impl.h | 180 - .../tbb/internal/_allocator_traits.h | 156 - .../tbb/internal/_concurrent_queue_impl.h | 1081 ---- .../tbb/internal/_concurrent_skip_list_impl.h | 1085 ----- .../tbb/internal/_concurrent_unordered_impl.h | 1684 ------- .../_deprecated_header_message_guard.h | 69 - .../tbb/internal/_flow_graph_async_msg_impl.h | 153 - .../tbb/internal/_flow_graph_cache_impl.h | 592 --- src/3rd party/tbb/internal/_flow_graph_impl.h | 547 --- .../tbb/internal/_flow_graph_indexer_impl.h | 480 -- .../internal/_flow_graph_item_buffer_impl.h | 284 -- .../tbb/internal/_flow_graph_join_impl.h | 2002 -------- .../tbb/internal/_flow_graph_node_impl.h | 971 ---- .../internal/_flow_graph_nodes_deduction.h | 260 - .../tbb/internal/_flow_graph_streaming_node.h | 742 --- .../internal/_flow_graph_tagged_buffer_impl.h | 249 - .../tbb/internal/_flow_graph_trace_impl.h | 364 -- .../tbb/internal/_flow_graph_types_impl.h | 723 --- src/3rd party/tbb/internal/_mutex_padding.h | 98 - .../tbb/internal/_node_handle_impl.h | 168 - src/3rd party/tbb/internal/_range_iterator.h | 66 - .../tbb/internal/_tbb_hash_compare_impl.h | 105 - src/3rd party/tbb/internal/_tbb_trace_impl.h | 55 - src/3rd party/tbb/internal/_tbb_windef.h | 69 - .../tbb/internal/_template_helpers.h | 284 -- .../tbb/internal/_x86_eliding_mutex_impl.h | 144 - .../tbb/internal/_x86_rtm_rw_mutex_impl.h | 223 - src/3rd party/tbb/iterators.h | 326 -- src/3rd party/tbb/machine/gcc_arm.h | 216 - src/3rd party/tbb/machine/gcc_generic.h | 233 - src/3rd party/tbb/machine/gcc_ia32_common.h | 109 - src/3rd party/tbb/machine/gcc_itsx.h | 119 - src/3rd party/tbb/machine/ibm_aix51.h | 66 - src/3rd party/tbb/machine/icc_generic.h | 258 - src/3rd party/tbb/machine/linux_common.h | 105 - src/3rd party/tbb/machine/linux_ia32.h | 228 - src/3rd party/tbb/machine/linux_ia64.h | 177 - src/3rd party/tbb/machine/linux_intel64.h | 92 - src/3rd party/tbb/machine/mac_ppc.h | 309 -- src/3rd party/tbb/machine/macos_common.h | 129 - src/3rd party/tbb/machine/mic_common.h | 53 - src/3rd party/tbb/machine/msvc_armv7.h | 167 - src/3rd party/tbb/machine/msvc_ia32_common.h | 275 -- src/3rd party/tbb/machine/sunos_sparc.h | 199 - src/3rd party/tbb/machine/windows_api.h | 65 - src/3rd party/tbb/machine/windows_ia32.h | 105 - src/3rd party/tbb/machine/windows_intel64.h | 70 - src/3rd party/tbb/memory_pool.h | 262 +- src/3rd party/tbb/mutex.h | 233 +- src/3rd party/tbb/null_mutex.h | 37 +- src/3rd party/tbb/null_rw_mutex.h | 39 +- src/3rd party/tbb/parallel_do.h | 553 --- src/3rd party/tbb/parallel_for.h | 412 +- src/3rd party/tbb/parallel_for_each.h | 120 +- src/3rd party/tbb/parallel_invoke.h | 447 +- src/3rd party/tbb/parallel_pipeline.h | 17 + src/3rd party/tbb/parallel_reduce.h | 644 +-- src/3rd party/tbb/parallel_scan.h | 403 +- src/3rd party/tbb/parallel_sort.h | 244 +- src/3rd party/tbb/parallel_while.h | 188 - src/3rd party/tbb/partitioner.h | 668 +-- src/3rd party/tbb/pipeline.h | 682 --- src/3rd party/tbb/profiling.h | 17 + src/3rd party/tbb/queuing_mutex.h | 100 +- src/3rd party/tbb/queuing_rw_mutex.h | 141 +- src/3rd party/tbb/reader_writer_lock.h | 246 - src/3rd party/tbb/recursive_mutex.h | 248 - src/3rd party/tbb/runtime_loader.h | 193 - src/3rd party/tbb/rw_mutex.h | 17 + src/3rd party/tbb/scalable_allocator.h | 375 +- src/3rd party/tbb/spin_mutex.h | 201 +- src/3rd party/tbb/spin_rw_mutex.h | 239 +- src/3rd party/tbb/task.h | 1165 +---- src/3rd party/tbb/task_arena.h | 493 +- src/3rd party/tbb/task_group.h | 353 +- src/3rd party/tbb/task_scheduler_init.h | 174 - src/3rd party/tbb/task_scheduler_observer.h | 153 +- src/3rd party/tbb/tbb.h | 84 +- src/3rd party/tbb/tbb_allocator.h | 190 +- src/3rd party/tbb/tbb_config.h | 873 ---- src/3rd party/tbb/tbb_exception.h | 362 -- src/3rd party/tbb/tbb_machine.h | 978 ---- src/3rd party/tbb/tbb_profiling.h | 355 -- src/3rd party/tbb/tbb_stddef.h | 565 --- src/3rd party/tbb/tbb_thread.h | 345 -- src/3rd party/tbb/tbbmalloc_proxy.h | 52 +- src/3rd party/tbb/tick_count.h | 123 +- src/3rd party/tbb/version.h | 17 + 224 files changed, 36041 insertions(+), 45446 deletions(-) delete mode 100644 sdk/binaries/tbb.dll create mode 100644 sdk/binaries/tbb12.dll create mode 100644 sdk/binaries/tbbmalloc.dll create mode 100644 sdk/binaries/tbbmalloc_proxy.dll create mode 100644 sdk/libraries/x64/tbb12.lib create mode 100644 sdk/libraries/x64/tbbmalloc.lib create mode 100644 sdk/libraries/x64/tbbmalloc_proxy.lib create mode 100644 src/3rd party/oneapi/tbb.h create mode 100644 src/3rd party/oneapi/tbb/blocked_nd_range.h create mode 100644 src/3rd party/oneapi/tbb/blocked_range.h create mode 100644 src/3rd party/oneapi/tbb/blocked_range2d.h create mode 100644 src/3rd party/oneapi/tbb/blocked_range3d.h create mode 100644 src/3rd party/oneapi/tbb/cache_aligned_allocator.h create mode 100644 src/3rd party/oneapi/tbb/collaborative_call_once.h create mode 100644 src/3rd party/oneapi/tbb/combinable.h create mode 100644 src/3rd party/oneapi/tbb/concurrent_hash_map.h create mode 100644 src/3rd party/oneapi/tbb/concurrent_lru_cache.h create mode 100644 src/3rd party/oneapi/tbb/concurrent_map.h create mode 100644 src/3rd party/oneapi/tbb/concurrent_priority_queue.h create mode 100644 src/3rd party/oneapi/tbb/concurrent_queue.h create mode 100644 src/3rd party/oneapi/tbb/concurrent_set.h create mode 100644 src/3rd party/oneapi/tbb/concurrent_unordered_map.h create mode 100644 src/3rd party/oneapi/tbb/concurrent_unordered_set.h create mode 100644 src/3rd party/oneapi/tbb/concurrent_vector.h create mode 100644 src/3rd party/oneapi/tbb/detail/_aggregator.h create mode 100644 src/3rd party/oneapi/tbb/detail/_aligned_space.h create mode 100644 src/3rd party/oneapi/tbb/detail/_allocator_traits.h create mode 100644 src/3rd party/oneapi/tbb/detail/_assert.h rename src/3rd party/{tbb/internal/_warning_suppress_disable_notice.h => oneapi/tbb/detail/_attach.h} (65%) create mode 100644 src/3rd party/oneapi/tbb/detail/_concurrent_queue_base.h create mode 100644 src/3rd party/oneapi/tbb/detail/_concurrent_skip_list.h create mode 100644 src/3rd party/oneapi/tbb/detail/_concurrent_unordered_base.h create mode 100644 src/3rd party/oneapi/tbb/detail/_config.h create mode 100644 src/3rd party/oneapi/tbb/detail/_containers_helpers.h create mode 100644 src/3rd party/oneapi/tbb/detail/_exception.h create mode 100644 src/3rd party/oneapi/tbb/detail/_export.h rename src/3rd party/{tbb/internal => oneapi/tbb/detail}/_flow_graph_body_impl.h (51%) create mode 100644 src/3rd party/oneapi/tbb/detail/_flow_graph_cache_impl.h create mode 100644 src/3rd party/oneapi/tbb/detail/_flow_graph_impl.h create mode 100644 src/3rd party/oneapi/tbb/detail/_flow_graph_indexer_impl.h create mode 100644 src/3rd party/oneapi/tbb/detail/_flow_graph_item_buffer_impl.h create mode 100644 src/3rd party/oneapi/tbb/detail/_flow_graph_join_impl.h create mode 100644 src/3rd party/oneapi/tbb/detail/_flow_graph_node_impl.h rename src/3rd party/{tbb/internal => oneapi/tbb/detail}/_flow_graph_node_set_impl.h (84%) create mode 100644 src/3rd party/oneapi/tbb/detail/_flow_graph_nodes_deduction.h create mode 100644 src/3rd party/oneapi/tbb/detail/_flow_graph_resource_limiting.h create mode 100644 src/3rd party/oneapi/tbb/detail/_flow_graph_tagged_buffer_impl.h create mode 100644 src/3rd party/oneapi/tbb/detail/_flow_graph_trace_impl.h create mode 100644 src/3rd party/oneapi/tbb/detail/_flow_graph_types_impl.h create mode 100644 src/3rd party/oneapi/tbb/detail/_hash_compare.h create mode 100644 src/3rd party/oneapi/tbb/detail/_intrusive_list_node.h create mode 100644 src/3rd party/oneapi/tbb/detail/_machine.h create mode 100644 src/3rd party/oneapi/tbb/detail/_mutex_common.h rename src/3rd party/{tbb/internal/_warning_suppress_enable_notice.h => oneapi/tbb/detail/_namespace_injection.h} (54%) create mode 100644 src/3rd party/oneapi/tbb/detail/_node_handle.h create mode 100644 src/3rd party/oneapi/tbb/detail/_pipeline_filters.h create mode 100644 src/3rd party/oneapi/tbb/detail/_pipeline_filters_deduction.h create mode 100644 src/3rd party/oneapi/tbb/detail/_range_common.h create mode 100644 src/3rd party/oneapi/tbb/detail/_rtm_mutex.h create mode 100644 src/3rd party/oneapi/tbb/detail/_rtm_rw_mutex.h create mode 100644 src/3rd party/oneapi/tbb/detail/_scoped_lock.h create mode 100644 src/3rd party/oneapi/tbb/detail/_segment_table.h create mode 100644 src/3rd party/oneapi/tbb/detail/_small_object_pool.h rename src/3rd party/{tbb/internal/_tbb_strings.h => oneapi/tbb/detail/_string_resource.h} (86%) create mode 100644 src/3rd party/oneapi/tbb/detail/_task.h create mode 100644 src/3rd party/oneapi/tbb/detail/_task_handle.h create mode 100644 src/3rd party/oneapi/tbb/detail/_template_helpers.h create mode 100644 src/3rd party/oneapi/tbb/detail/_utils.h create mode 100644 src/3rd party/oneapi/tbb/detail/_waitable_atomic.h create mode 100644 src/3rd party/oneapi/tbb/enumerable_thread_specific.h create mode 100644 src/3rd party/oneapi/tbb/flow_graph.h create mode 100644 src/3rd party/oneapi/tbb/flow_graph_abstractions.h create mode 100644 src/3rd party/oneapi/tbb/global_control.h create mode 100644 src/3rd party/oneapi/tbb/info.h create mode 100644 src/3rd party/oneapi/tbb/memory_pool.h create mode 100644 src/3rd party/oneapi/tbb/mutex.h create mode 100644 src/3rd party/oneapi/tbb/null_mutex.h create mode 100644 src/3rd party/oneapi/tbb/null_rw_mutex.h create mode 100644 src/3rd party/oneapi/tbb/parallel_for.h create mode 100644 src/3rd party/oneapi/tbb/parallel_for_each.h create mode 100644 src/3rd party/oneapi/tbb/parallel_invoke.h create mode 100644 src/3rd party/oneapi/tbb/parallel_pipeline.h create mode 100644 src/3rd party/oneapi/tbb/parallel_reduce.h create mode 100644 src/3rd party/oneapi/tbb/parallel_scan.h create mode 100644 src/3rd party/oneapi/tbb/parallel_sort.h create mode 100644 src/3rd party/oneapi/tbb/partitioner.h create mode 100644 src/3rd party/oneapi/tbb/profiling.h create mode 100644 src/3rd party/oneapi/tbb/queuing_mutex.h create mode 100644 src/3rd party/oneapi/tbb/queuing_rw_mutex.h create mode 100644 src/3rd party/oneapi/tbb/rw_mutex.h create mode 100644 src/3rd party/oneapi/tbb/scalable_allocator.h create mode 100644 src/3rd party/oneapi/tbb/spin_mutex.h create mode 100644 src/3rd party/oneapi/tbb/spin_rw_mutex.h rename src/3rd party/{tbb/tbb_disable_exceptions.h => oneapi/tbb/task.h} (50%) create mode 100644 src/3rd party/oneapi/tbb/task_arena.h create mode 100644 src/3rd party/oneapi/tbb/task_group.h create mode 100644 src/3rd party/oneapi/tbb/task_scheduler_observer.h create mode 100644 src/3rd party/oneapi/tbb/tbb_allocator.h create mode 100644 src/3rd party/oneapi/tbb/tbbmalloc_proxy.h create mode 100644 src/3rd party/oneapi/tbb/tick_count.h create mode 100644 src/3rd party/oneapi/tbb/version.h delete mode 100644 src/3rd party/serial/tbb/parallel_for.h delete mode 100644 src/3rd party/tbb/aggregator.h delete mode 100644 src/3rd party/tbb/aligned_space.h delete mode 100644 src/3rd party/tbb/atomic.h rename src/3rd party/{serial/tbb/tbb_annotate.h => tbb/blocked_nd_range.h} (53%) delete mode 100644 src/3rd party/tbb/blocked_rangeNd.h create mode 100644 src/3rd party/tbb/collaborative_call_once.h delete mode 100644 src/3rd party/tbb/compat/condition_variable delete mode 100644 src/3rd party/tbb/compat/ppl.h delete mode 100644 src/3rd party/tbb/compat/thread delete mode 100644 src/3rd party/tbb/compat/tuple delete mode 100644 src/3rd party/tbb/critical_section.h delete mode 100644 src/3rd party/tbb/flow_graph_opencl_node.h delete mode 100644 src/3rd party/tbb/index.html delete mode 100644 src/3rd party/tbb/internal/_aggregator_impl.h delete mode 100644 src/3rd party/tbb/internal/_allocator_traits.h delete mode 100644 src/3rd party/tbb/internal/_concurrent_queue_impl.h delete mode 100644 src/3rd party/tbb/internal/_concurrent_skip_list_impl.h delete mode 100644 src/3rd party/tbb/internal/_concurrent_unordered_impl.h delete mode 100644 src/3rd party/tbb/internal/_deprecated_header_message_guard.h delete mode 100644 src/3rd party/tbb/internal/_flow_graph_async_msg_impl.h delete mode 100644 src/3rd party/tbb/internal/_flow_graph_cache_impl.h delete mode 100644 src/3rd party/tbb/internal/_flow_graph_impl.h delete mode 100644 src/3rd party/tbb/internal/_flow_graph_indexer_impl.h delete mode 100644 src/3rd party/tbb/internal/_flow_graph_item_buffer_impl.h delete mode 100644 src/3rd party/tbb/internal/_flow_graph_join_impl.h delete mode 100644 src/3rd party/tbb/internal/_flow_graph_node_impl.h delete mode 100644 src/3rd party/tbb/internal/_flow_graph_nodes_deduction.h delete mode 100644 src/3rd party/tbb/internal/_flow_graph_streaming_node.h delete mode 100644 src/3rd party/tbb/internal/_flow_graph_tagged_buffer_impl.h delete mode 100644 src/3rd party/tbb/internal/_flow_graph_trace_impl.h delete mode 100644 src/3rd party/tbb/internal/_flow_graph_types_impl.h delete mode 100644 src/3rd party/tbb/internal/_mutex_padding.h delete mode 100644 src/3rd party/tbb/internal/_node_handle_impl.h delete mode 100644 src/3rd party/tbb/internal/_range_iterator.h delete mode 100644 src/3rd party/tbb/internal/_tbb_hash_compare_impl.h delete mode 100644 src/3rd party/tbb/internal/_tbb_trace_impl.h delete mode 100644 src/3rd party/tbb/internal/_tbb_windef.h delete mode 100644 src/3rd party/tbb/internal/_template_helpers.h delete mode 100644 src/3rd party/tbb/internal/_x86_eliding_mutex_impl.h delete mode 100644 src/3rd party/tbb/internal/_x86_rtm_rw_mutex_impl.h delete mode 100644 src/3rd party/tbb/iterators.h delete mode 100644 src/3rd party/tbb/machine/gcc_arm.h delete mode 100644 src/3rd party/tbb/machine/gcc_generic.h delete mode 100644 src/3rd party/tbb/machine/gcc_ia32_common.h delete mode 100644 src/3rd party/tbb/machine/gcc_itsx.h delete mode 100644 src/3rd party/tbb/machine/ibm_aix51.h delete mode 100644 src/3rd party/tbb/machine/icc_generic.h delete mode 100644 src/3rd party/tbb/machine/linux_common.h delete mode 100644 src/3rd party/tbb/machine/linux_ia32.h delete mode 100644 src/3rd party/tbb/machine/linux_ia64.h delete mode 100644 src/3rd party/tbb/machine/linux_intel64.h delete mode 100644 src/3rd party/tbb/machine/mac_ppc.h delete mode 100644 src/3rd party/tbb/machine/macos_common.h delete mode 100644 src/3rd party/tbb/machine/mic_common.h delete mode 100644 src/3rd party/tbb/machine/msvc_armv7.h delete mode 100644 src/3rd party/tbb/machine/msvc_ia32_common.h delete mode 100644 src/3rd party/tbb/machine/sunos_sparc.h delete mode 100644 src/3rd party/tbb/machine/windows_api.h delete mode 100644 src/3rd party/tbb/machine/windows_ia32.h delete mode 100644 src/3rd party/tbb/machine/windows_intel64.h delete mode 100644 src/3rd party/tbb/parallel_do.h create mode 100644 src/3rd party/tbb/parallel_pipeline.h delete mode 100644 src/3rd party/tbb/parallel_while.h delete mode 100644 src/3rd party/tbb/pipeline.h create mode 100644 src/3rd party/tbb/profiling.h delete mode 100644 src/3rd party/tbb/reader_writer_lock.h delete mode 100644 src/3rd party/tbb/recursive_mutex.h delete mode 100644 src/3rd party/tbb/runtime_loader.h create mode 100644 src/3rd party/tbb/rw_mutex.h delete mode 100644 src/3rd party/tbb/task_scheduler_init.h delete mode 100644 src/3rd party/tbb/tbb_config.h delete mode 100644 src/3rd party/tbb/tbb_exception.h delete mode 100644 src/3rd party/tbb/tbb_machine.h delete mode 100644 src/3rd party/tbb/tbb_profiling.h delete mode 100644 src/3rd party/tbb/tbb_stddef.h delete mode 100644 src/3rd party/tbb/tbb_thread.h create mode 100644 src/3rd party/tbb/version.h diff --git a/.gitignore b/.gitignore index 79ddee2722..812be7c3b1 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,15 @@ compressor/mod/ # emacs temp file \#*\# -*~ \ No newline at end of file +*~ + +# TBB import libs must be committed so a fresh clone can link. +# The /**/x64/ rule above excludes the *directory*, and git will not +# descend into an excluded directory -- so the directory exclusion must +# be lifted first, then everything re-ignored, then these files re-included. +!/sdk/libraries/x64/ +/sdk/libraries/x64/* +!/sdk/libraries/x64/tbb.lib +!/sdk/libraries/x64/tbb12.lib +!/sdk/libraries/x64/tbbmalloc.lib +!/sdk/libraries/x64/tbbmalloc_proxy.lib diff --git a/sdk/binaries/tbb.dll b/sdk/binaries/tbb.dll deleted file mode 100644 index 3e5c8c862e7f7c5ce96d96b2a5c0b71518ecf8aa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 391040 zcmd4434B!5+4w&YMgkJvk&&7}g7?B5h_aX}n^hF83k@@XmS}Ji z;d-5xzQxwJuWfDJ+PAK6ZL3ymLXfbigjL*epE07iluc#+-{+h=lSv2&`1bdI`DpHT z&U2pgoaa2}+0L2Bg^PT7KA*1_|EJSF-(y_mU%C3b?>~h-e7^q2KGEOzXrJf4`B)G8 zxo?i2amB3A%-U-&uf6!{&?OgNbIrBU&=04FYGc=guDB*tId)v=>T55Zeo{d}-y#q6 zPyK&du=3K@^_l-$2900e#PdTvmu*-sd;8n681p{*NMspqowa-BT%8TEYr6_?DAHuu)U>MEb_(wqDGD(-%G za;D5y-?u`C_U!NTJrBi7daBn=&v746xAl-i-}C7Z^{V6Qo5mZ6oM=15*V8wRqPkM% zvgfmYQ*v7`-=)KSzQr%}@;xdr%MbC@$Y9O%`=TScZu0x;kL6l@h;J^aH67o}H=f6? z{~r5xFJCb)LtGc!Tuy&%$J-8fUN)$WH)sNNV%qyX6k~fkMz}NQtkwe<@{^)hQOf7*BYy&3xd-L9l!5m}3ruI{6T)!GUp|oMGo2k~d}(pTq^UsDfQ)17 z1>m@f3pBh{mh%E2#{<%K@~n6|e@@VB9Z?vf5KlekjCfpztox&7nE1=S}^;8xLLXG9?R z>f>;lGa~`o;`O#C& z$DiXLPU0%MdhFgP>J~#)Dg}#N2!{eq+ARX!x$Q;;2s8iMo6>;*9kZnPG|~X42S^ zmU97Jt_wt*bD#oEf_PPknstS(jf;2!jBl&)u+h*I9cw$&t0T@*+qt^Ny-hyGcLi=d z%DiNW^rQEui_G-$*v`DQj{4SqNILit!qaP6!-%!o?$tFBXS3~2uTGx}&8kCIcw=-t zsJYkEjZyh#_$lf7oU^R78L;JrJJfQ{G85MivfZO?_k4I^T>iq|kSSoMt3dU7dN5ST zVL1aBItF2-6@EVUpb$kc%UdRttjf2-t5OenVC1ve71;fMLn($8g8mA>TIP!BMS(EkC7i-zp-n8>SXg?AXY9xWr9P>ksM z!ip86M|)5NbaE$99v5+(G%7rJx{cDbTM-IZm6scfTN~5SBAP0DLPPOY>J|$yaum1LkUMg%VUsXSb46pG}NZxX6rW0pDZHBI@ z&>q6fsAfQJs|SMZCgc(cHyepkTE9gxSVL26KyEKerOo=fbK&8QU!>FZbydFTVVyBv zxRRlx59#1+?iQsUq?`uU{z%6>zp}(kM2eH2GDXb!OJ(j)Zt3*Zi58WZ&f=mZpG-H# z#QjA2l>B`m&rUOs^dXR3GQ{UAT~WF=+0sI@3IC^v3e%Ok-9yy?RgItP8qHE8Q`nRL zrrRUKn~1M+-vm~JX(F6LR($i0h{FODanCEXn%ncN^h)Mq8FHu^_O7v=h)ArENG#i_ zn#O%GJ@7@2d&zbtm&A^>+#a^GB3)g$NHL{bRb0B3o=h{H>U`6I`zKV}iSr9B<`sO^ zm?wPI*km;RCqS4+HIc>@M#IJOHok~#EYj%WJgL*pazDc)S7kX4Ir z@)yH;_*kiLJkq?R6&z+~a1iKu>Gu25!53SFeXFLm|5x{LQacrgO)i8f1A>Q{7{yib zO9{`&8K%?#Kq!(1lz7u)nmSum^&U=E4;c&sn_ zz4{vmD=9KI&>{DRvv{JxL0SLFavNV7nws9ME-jZW5=rr};4M>9ltoWjm}xqRJLSD| z+aKl9S~UX_q?u`m@_{~ADNRmQz+;a{hL0oEsLD4JKd+ZsOsXMmX5(7pXEWib`nrL> z*iqoo`dMbZaT5YP-3H7a9V-s2k5g67|E8I!nzV32aN&f>3nxrjIAJPyOrrv;P0hjp z(z%>rlM%~jR^}%^I!cLVSw8A1WV&=3WIPm9Ke7u;RUmq#R1_e|ds%e!s_^7c4+NjQ z^`s_eJ*n=X>-$S(J(-eSPi~^7T2IQ87XXg+q%K%r7wi!=6^?>y2G4W%h38LeWX-4w zvS9SjBIp;0^>tHy(Qnn)P1Y+#4lYx`MHo08wACG*hAx)VWG2q0+4s;Q)G(r0J&j_u z!pCT^Ox+2Ql6?^c4NXSFsf;gF1Wr}H(y<;}C^Xb+mL+0RD}3FmS)PhzB^HTxyH`z1 zAc|=5G_A>KTE!R`Xj&x^=NWX;GB+V*;yabX)!a$At#H&g5pg%L;OwDr#ma2wO%$%; zh?7M9Dp7jZV|N114DX88ls4J!*pRYi!fJt@Gt5dalqKJGQ4U=r;`HUO?M_AW`a!;( z7(1w9VGq%_ME$ZIv4PG(f}?&7MI zQNg5ny6(O_Fda}U3 z`43fiYHs0-JP~;yNM)C3y(F(@NLi6%)eMzsIU5+b@`&>+YtXcR%78L2OO$+Y&qe=i zihe)hylXjM*q98?RMz;(VrI zy5|-GEA~~ee(b?21XgTMShjmk#93BV`f|h_UsPztoARyZ4@KFC9QR4-wn+FBlnY

<*pr8AQ zJFv)h-msnbBMzF)#}(=G!DiUUA4xG7k7Dm~-yXDFn~r-{YGzf1V*`=KEk*+t9dpn# zWz2YMNVI@rU&WJ(&Q4v(n>?q`R;2N5P1zN(osqn^g|e5h+;o~6QPhnu0ynH_f6H>~ zfE+BAAntKTD?({$}C^SQi63R-hO7 z_|ln^r%q|{7|U#}EGm(@cA`>9UyigKQ;6?E4N}Fj@32`L~jcD`w%1~|COpn1p zV;YKWJ(Rz;dm(dtYW}j}gJ5~#mo#0`wslV%0a0!_hlzp+BVk>yi2X%|+sq8NJd`1m zjD~k)wRE1*GQK1od|L!C^CbMQSbJ(YA4{83%eai2s6A>m44;k1#&lL8H!b(HqKJD( zQN7S`$ZL_tmyMtP68bZx&BR1JEn6+;Q*Zc|S>aD(C`7yL#Gu(${H@-We0`B&`>)Z7 zTVGMbHgZ=wjpyQz5$6rd`9!VWk7rl!n2{O78XR;}F3KIn5D2JIWYNU?l=x3szV|ke z(`4Y7TMzrY&t^I(xJe71?y_ z>lHTVp*GOedAvEl#2V6M4q1X!Yv7xB69gTXD>mWBII|=SP|bTFl4L3+3O9h?)Q&1*Q-F5)0tewWUqv3z_7Yg{y5@gx$r3 z0g(nQS;v{-;K0Ge6VPv%`#UQTP^wI8gDN9%%6Bg_<1NMI^}F+HpH{{JjDi`W#xHg6 z$vyCvioDLX7fQ6-YIvdT4iBIK_@EUQLWH?9A)}~+E2eG*vrxn%Qi=XEsvR`29FLfp zx8)_4Q(hva#KY+s8p<66VOp!I1zSvD`~)#Zwyv1vI)qk2D>0UCmT)&#ZYBoRqnDWx z_Zpy7!`Kb4DxKG$u-ad=T9}#yr=2p@zQq7 zB8_RIVHVml_8p6#uwovr1NOaR6nf(*zWI)X>V{T5F%%y?4(oM)JV{+VG}OzuLsaceuoNDdOKKBUzB8-;`! z+aT*d5D;y`fD$i2%gC7jwv%{RF@Y>uw>`#frPJ6X1+$)jNWcphVE=RtzY5Gzpay9%4VhYJ+m$;Iu;niX1UYc?A(C|C1-?-7Hn zDO-(5*}sT^*s;~xjzudGJ9e6zkOgaRcC368ee`dSc3B}MaDb+*>{wRDB+O8wR>ip? zF=8ilGGb+gpEFd)559&4i_+e`1v@Qk!5#vhITkD*vKH)}AfbW?pS;<1@VW~kOhBX^ zKm;MsC4x|puiP_C4&q;pibIF&K*GaixxRNRywkXKfDCkpzo)Qs6yYyG(k`DB#TusP zo!@s9;j{ng97VX5%3c)VVjzmwXHb3JARRvt=Tb2fGGOP#4@UL&nsbRCs4j@_YVRS0 zvzWKHR1NAHKll_dUi@ImUg8HPxY9cDgEP2Wl=`*A52^;WKiV4Q8p|PBcikbQEVhJk1Y3suI7=I9?MMncbG?SE1lXik6^%OWDO!oApB*lJ=L1g zsWsJ4>&$InPx&ODh+yju3)Y@^(somGcJn@ZmDzqY{h1p+^3AMzef( zt5%WI`L*JO4^>Ro_LSEFVK=28%Z*x3SwG5H41h=s{%SaYrC~aKsSBZa7Nf+LFrUiN zn5Sj6XHk3cB#57%D1zyhqj(KR<*6#qXrME-2Ff=nii_O6p6{ZK)Nhz5%T5Zcgxa-Ff0cECJtn?Ba+rkTi%KHT2sa;``D=k7=*-N_DG<3L?46XxDBr4xKd4;`W?w zb^WNig-YsmOSIB^JG4^Nj31&O!Gtc%NHzxAClgF*pG++xk}`6M>D5a8ghjO@OTBHC zf$+6A%UV&-ote-YsJIVUC!o#*Gg;ryTNHkLk6IKKX$@4$i}z2iV!!H)qfWNu*DxBz zw9L$Q6{M;s)WbCsa1oH||na~;YK_7Adp=6>Z%$+-NR@}>SkBB=PN zzE-}}FJDvEPl&nte=%R`>HE6y&;Ox(sc(Q=#?(P5&cGg<*)?CP_r|Vc)SapG_4B2U z{ai^?hVkpmd+d??svY_g}XAH_*(f=dD7;- z^QCHJnNjx6NbbHOU#f*Ek~KOyq<}1Ad&!sj@u$@2$d~#Ll6aLB^NZiA)$IWCr7qSr zy33aur*vO*!*!^pYH@m^24O=6Tk@Mf6WxaZJ<{Q9NiCK13I}zv1}~ysn{9kpr7xGR z9pm2GQ}kBbu{RZ>*WflQ#`T_G`>yC7rb`W%rG5koi+ z=Z7MBn@r=Ob+$%OLU17XQY|rpRI&uut4*{_*7nKb?Z&?i{VOIa>qCr#4Xg&82jPx{ z48kEY24{kwHwLyd16{l(AVKD;{C(4()Udsff21b=JNr$3?7K~ck-S$G`KK_L41ps5 z9{_>V7y>}Jo;I#qC6zwk+PN9$yAho-P5ikSB04rbwszb(r^Fl_Tro@eM1i$q&?7Z=^h#da%2nuFqN*j(p;{}wnRpg1V!}Aki_U>`v#F*v zS(fRQ)^S8kR|;cgruFzX$k2C`W4AG_dsqpRSq&eM(kE8AM8j{Yr|*n#tdG@Nq1eAg zLbN$EJH^aaQUma#^voX4%qB@ZuLDLkKWtAitSc&UGCZzs4`33JH*u9p#;CF{-MxxE#=NH7A`p^YP zgtHu;{-)^$r}yx~$N`JMU#ENnH6U=9zmfvV3lbd4UHit2|D?kplYGATn>#F7NmfMM z>cY~djx1Xfhl7_{8-aPNh@w@Mv{uPdL%U+`SC#2R+w$0PD#4){o?axc(J9L$(N^kZ z5^W)##ZdPlAF&}8Xl7-wl9G)I@z{FyAxURz&AJ9_S=W;HQCmulWng856J72WLwEk zHuF8#qz5PUF-|v~54%UQ?Xg00rAcIxZHolyYveW=d zc5vO5Aqhl@V9VekT}cKvNb9*7w#V^CsZZl?Q|i+Rg5Uh5_0-fUZ*qrQp?i4Gtk>|s z+?OoGT34X@XiyjG#5N5oGchv$2G(~KF-K6V^?h~c739fAS|wk=IZ`6$Eb)>`$r~2Y zD_P<%%q;Ous8}kO5f$qttztD5XH={swLgx8k{k`IC3cwAdajajYSo)6tDf?DBnFj> zie)w@k#(Mm)nvPqUP6mGA6a)ETFg3RU7MY^Ho``hi2Ea11D-K%`ZF|&xCW8~S$Y(U zfT(LWPD8DxS>d)=A)fP#L`fR$RyMH=IevIK}erxXg7&y6Yu)VH!`hDc@(KWZkBh zn2FK(rA;b6z#tYrAwhaGF(UMMIX8Bgm8WuLDdC2GZB~5eIkWSnN_b7}ht3+Ez;CVF z49$1j;bpTflcl{_MF$bdmh-ACnQjJu%EB}yR6&AvS=ggmzKyI&<{h+z1{l3cB@_imGjAmDoU{fUz8wLpRtEMB8FNG86XD z5Go3Zkg-Zc`0iy|0{2X}eY(iRuQp9>4v_K!mpak$Bv8IbG0B}AK>Z_8Pp})jmeb2B zlnf?SBHo&Bt$P!DfS#?Zy;U-Js%8I)sq?f1B;YQkpEXRI@k7CSZR`({iiB`_j!=!{ z`vpE-1nb9kW*ac`wve3y_N+(G$k@4+Qj@2@OMfM7vNRn$=~p5>J)GK~SEH;p1Bl)A zr};vdC6Y=MdaPV(k^WZ9mBnYR?Afu4P*_!-5lKmMjW2;Amd7^hw+gOG8cd90n3j}D z@KRF29olaq?PFzZivXD>l|!~LGpm*DX!avTDGFG-D3q)hM5OzYH&vFp3yBHQ;s1DZ zsNykUQH@@Bjb&6evDeV;wbF|3Q0&=^l2!Ymo@lI%@TRtRL>3Y$At?=+hf>sJB}NDI zN-4R!Z%X_X+^5;ji#8h)Bf?6<>F%zi?7)(cj__TFz9lQco$^KY6A=HHMw%Gw=Uf$T z*(y_w5QrH`T#&ZnD^Uk0&t&RtiWpC!$X;EI&5IANW=!I=cYp!-x^gnuUzK}rJ5U*& zW&`>BO#?RZ(hnCPCz7Q0k@s?(%*Nfo&$cwUAj#Q zL-IL&^)8vK`}Sb&K5NC7?Evtc`>gOQ#)5y)roguI=!t?o*Lg8_XiUV$d4T?1OBc`; zIx$+<_u*bbldX}Q-0FzVt-g4%t@qwy9>S(077zWrS!11m%Z|G)4>;XkTtBf0oL z&<7bQ{sSf_sNY-#^`RizJ3;O6@>v5QVL(Wz!4|iFx6)bhv?w&@y7!2r?uJ&y=68uL z3?W;vN>oAAHFjOL_jj7J>USy34DWFr>u7;MQH zuf4rlM zl+wFooSld!8ncZ9GO|$xD#-;RS=S9sl>)tiv@9Obj7q2~JnK^_tQF1XR zgh<|u^>v!F1)W%7O05U3{~OI751P)4qGRWEu;Aemi6mbxhXqZ|g0KBt=t(X1kR@i`avW4GEQMDt?+Gq?_)Y3o3>ZeQ(wXRuX>@>HXng0)zEB1zK9!n`gRx+P8o4UHWx!6H zumhb%6Cd0p&&&V%(=?y=DdQYPYNU`FPE3!z8tmG~6e|6NP6FAlAKCPVBwd|?IkJ|Ife~;HTsq8O)=HVfpiBQT!ojhUow@I zHiM4DuJ0`#4pYXRYTMZS(*;TnPf^(vXT#f0zG)bAu zBoKWL275o%ueDKfCf`V}U1T|@6~Wfj2Y0soAgObtMF?qu%&IHV(N$%EB!~CHzy4Qr z10|2!^mG<>Byc9CbpO?nM7-?=H#hF>uZ0S$AZ#Z>R>sNVZEvSeZc!FWp9Se zG8YB!7*_9K-#Lmj{adU z=FCPa3dGPwYJYC$Nvd9F%u=G?;&IklOjrr4)#zg_Sc$PYdf1K%Ns$C3`jhe0)=KAN zYu!6m{GHvFQ^xw9SBuiaZmGnz7(&k&4a+GjFGeL!fbaA)zT6If6deo7)<3e76VW4| zbJU7@YId(y^u1Mk7-X(YRyu`<1aW$PLNJ~6KRFFMT&Wb>pXk~`u1@iT9_U0vzC=9B zlI64VmO`mw5~wP01j`X3;P_V!A@33j&ye>n9nwN2 z)JP1O=`#=n2!q%((LYxK)M9C<(7fKigz`)3mC+MO@tZCVHJxSd*!x`atC3N zfp5N*US^~8e8|q4+7lz;cP+O3;Qv(`&@@XHTs&%koq?Dx=HtwfXtGAa-+gaV>C+;kbR*p)G7 zkexVtyQkqcXbor??`J2aB&e}#P-{SH{6quNOEv@(>r(dOqsrMsR|0H9fgdsX!VjoPx*wRV4fMVY4G6ykJh;go zvfRvD6)_%aPR_sqnHludJf;v6Y|`I5^{KVD$Vh>zG$29^Xh6lj+5L2!#b{W{=LkC~ z$@!0|kwhNDD_c2}=bYJ>$neB3rc*4zGDOCLw~8i*o~Df>Li0-O|KvQ6dEV&lR?)I*GHl(wmpX5W;{L6F&BaPHCnf%>2H(Fn^Hp(`LeQXYhd@({P=4|j49%?iqxS1dWLH$6 za_n~7dEk#iYUiFi)#W#BTu6rAq@Ef#t4ks&FZEGBG|n_FW zE%we>9K&yd@p$la`7kMwBodFqM=4)Ul28eTZ`KRQwSKS&~Ezsd=> z!p#g=czx_B+r8&b5RoK$MKwfK6VJo~@bMu9W|EM-nHSv7Rj!$=1~-9s0eL0*2OOu1 z>4RRp=$;jRmZTj#X^naZuR8UGP4+?&XlKM5z&tt@O6@cNP=1$~p!9dfWa9bihRt;up!0MIt`9f8hdnO30;32WV3$~~STiy}KB zt(t;DxKfyDttg~^w#Goal60CdMiocEh{Ltn|-W!&CI2t2_tx4au6*!9!?$sg;2EvgVKgP3Ks=d{I z1l<$DcX>~z>UkT%+Ynx2+;kyxL^mvC*1TXP`nB2N<|u*jfF*7_%@4d1@13fSk^qq# z*Um1>Sx8v5Wz|92X-;<|Gwr^Rq5L_ zs05Yf9Ox--JUzy)bb)3=FG zU;}wHfOAs&Tc}>OmbjRS07+D;F-2KE8iV~}`M8%`tI7oHy zhP``#%<#z2tfNf#5Hf9cTLTnOMsO`oMZ;+@$=nP5~f@;PB*(7a( zDo6+hOrX}0AwgI0DBz1Cfpdd$2DNIeOdj#3twRKjL7l}8>-V|RgdPASnh>>0BF6A$ zCW-OT>-LZ)G$G85*5s7G^@PLpn6=ku2=DsGkQ3nP57IB~O-bD=YKMc4bpyU2>{p?8 zjEUs$-MP4XxSA?T-|fk0NCF4^xfeBJ$qhn{!&T{>^i!!4D2Cp!c}mu752SdRb|@w5 zQsIO64w-2RzPOqm5<0p5pFPlvy(TAo?(Eclopo;!PZW_erNqj6U3JV;uqJCa6MO|{ zD0V^163fXwCiNSc7}CaKZj>kMaWz%^C4h%RKi`8wie+=AX>&gqi=)=b&nQdgJ#0XZ=2pl$nwU!qO>)1|B$aKtN+c8TBbJWW_?;`d)3O^C9HZpV&HDof3#<#s@R{xAm zD9H`u0?dHS+Qz1PtcDTA_IED|MTQVz8-cm7Hr5Yca<+`^T!bVzG{PBL@#h5v_ad5! z7BQuksjLzjw8fdtfsV#QoaE52d9>SWjWQ3}jUh%FJ2zfPlXNu>e+?~=^#VYFqvSYm zT1SMWCldD}BE&vnPPA-4Lao$RVmQH+5dqIEBw>b?;1Ge;?nwL$a$uce_lh41tY{&` z`nd@_*3eS1mA8rjb4!h=fp+F!ekOx0dB9ZVgQ38t2=Elo@Lo0YVdl>p2`5+N-&6Jx;AmiQwiCcxz#?NMUs zqdjnrkYfe&*v zXt+a)aejoe%kx>>&(1fTY7(9ZH`&IBCM$1;h!;eNL>v%JbzBFgE7BX-NTC$eO7Z8a z;D#Cg+_-6$@VBfd5{oStrY@5NsuD?{ViaX^5wjzk>?;b1h~wCjk8xR_`zDdS)OY|( zep#-E2uI>_+2%o%)iNY_;#{D20#v-^mcw?qlYQu{do9tgPo_a}4M};&e zK~kiUC@}7qfiIcp_nBzb(;o>D*+S$Ff>Qyr{dL?GR^AsFlwX>8pNn-x%wT$^)j7W) zp(&!z4@C6icR@$a1k*9j5SM!V(Oz{llNGs1_R-%wL{=@hC~-cLa)X_i%&<<9FgZbI ztc+(kB*T_$x2c7Z@T)S&yBTE3ERt7vQ$7`MigEM5)F9*kLj9dWlot?2gRHyqD8=Mx zjmGygqn`0p{VkPF<3r2N-p5##m{;4dds-sS=k}17?eHd?8!cwIeNL~^m)p-ax4&%W zy-H|B0Gq~$O}Jb>6r@tnM!N%fzN6?)AXTJEkKvs;qy@^o0v%=_C-pLQSdKWG zOKO-Tc2Mb$R7$VI)uKcWn3-CHTjAG@o3_ge8DCn~;UOc;FK&D1U+7{`yvwId*A;~p z1mPkbi+p1Xn%N=3t%}Lsd>x0{24Jf^iHtuI@mPjLe&>0lcj;2;SqahZ5XzK~YZ11n zT#F?K)3q0elD7g>#BhmO`m7l)cz}8_zo68TTr&a1E92T^7H_nim1OCXGBQz)`r$N1 zCCCsks%IEsd=+=jyk#M=HRC(-=Do!fK$#tx zZ#l!+(sKjq;q`?)Gb*xgkovr2?mrjn6>+A(N-eQomUAVa&ksbLF+w}9@l5|kQ(Z*o zGV$wbT(bBQr>XOWt|pa&DXLD=HK81a^o@i;KQeBX(>EjGR3^1#_6H&jtFqEi?Nak* z1NLlYHgL+6PA%aagHVN+TheBGxg~=*b67lZV%2)NB^jMe@;@Ts=Vtv77ZD4foCR4f z9!~KD7ew5#%!E}&!|9BYJdmdHaVE6{Un-MetJb5E555aGkb|M%iceW(mH@f-QT%8y zLaaRETub8jSg3jd3l6eX&F?1MR?OcCEW1Rs!b@fi#N5+bCZDcm)vF7&pCIh%G*_rK z+{=iG3`tr;kQBOzXcR;Z@NOD8nKSajcK>>(41@dB!(1XbwQuqRTPJp_j$eT>>H?fklFq8@bs2SC zXdi<*{dVivtxj!;N?%`%cCsY-;D3w0t_MuTZh7DZT@?7l98YSz-dSqcX>b&5Nw>hJCVTa0@xzMm|+5@^$_Jnmn04iYLx<$Cdy6WHNWdr z*geAQ4SwgW7=MX4MAq+Y>gevzXUc>;LRQOTGSS^Eic2MeF1Y?fH6k+GKetmuf{QBg zcQtqY7j}F_DyT1G6p#q)yfXD$Lh#HNw9w`kL<{|g^ZmJWA;q7yV;7<`Hl2P^iDc_eb zROTaz<&%BwuVyRCImO{ma%A*2=VPsB=OjDH%EWo#I^W4uB6=IiHEXg3$#zO%SoGPznohr+ceeRL)66tKrYA?Lx4T0ct~zs|N8%eK#4&*7f5qKCYJ~P&4so? z0Wx*9+oYOE$75NFJdephZdHq^aTHQ_%DZw!rtA(SCY2{%fgk0`GRq>WipH){oVBj~9eJr%MCo4RH--Mv8Sg9?6!T}q zw#7uw1QXRUtty!KAz}1*@>I@+=c&8-w95V&{@BL4OztPYx zGFR91~KE7L|&Da?SC6PEB!gCYvM{rR3ah9BH1I)xxz#jV?*QdUUb$ z)#$$9jc%Jax=W0P>(%IDfOI#yFsK?`91upsYHxH&8E15b5T-{6L^-}twV7>$g;ykJ zgex=+x(xA|ujdT$mf1vaB@Z2fm_i`NSB_@Islxc;n)1)Rz02rEFun_Q`c{|W&E(5^ z!>iJWPPszTh^Ubn->a3%ySDx9&f_anP56}YWl&XjK2$yHZhT*vA>%tHXLu!BMzH5h zAKUqt>O)FSKB5Dyzw(Cn!<^xLQ{5BoC#6YmBaDcP<-(98U~@oG^5<)1crRDOTb>zT z+4tB7xpFD(r#H7OQuchJ;9meQ?=%tXOeD@9)70N&P|7d`xz9%j0^p=n`HHjt&4@G7O zJvqduvNvI3`c3-nFtY4?#)3lrDx)_YyzufoA35p?viQ@%Djx8pVJ`)T%dMoaeS@;h z_lGO2`Ieu=N6oQ4l1Jq2k+0yWa}gIc_2 zZ%QuFn)jI^;-AplABeTfpq!3Z`&dHEY%W^dPijUSqQR4nEu?x#-jyj-n(tb-|R)X^(raGopf}*?Ht|?LBVVsn3Zm0boE#$Z%vxywHSw) zv%6-g44{r(buRO=lsN*FbBid{+wV($I07yU`T)JC{chz)%JEjt{~)QlfQ0tn1{7hB z(xv!;gdsNb2_fYA^BB))}Wmk%3^<1&YB;Z}_genaCCL)HS8HW(ro<>EBcx(rE9g#45e7}P9Nji zr{cfDz;Ea!Rcv>0FH@?r2{hvL7~@Vl72X!7-r*b#R@=sT(&xa>%Oj+X9M5JHvw1W8 z+^|&Yj&jGBz|TF~`ze&-m*n|aXH*xek%6qS(Qt%>+0Fwuh(to8@d64w1`LVnb{W5P zumVW!e@C{IKzvn-h|6%Eq|!WMjV)5@VrRp%rg}z(eI1;5Kpu zcvfJrp`}d|c>MaZQLZi2MPXyxH$90Sllr|ZK+>X)7zrQr2h&BV7f~L!x5_jz8hg_F z)Er8>{4iJ~7F?T@9=}(1GX(zn8-YZ_4j_rt6d+h98=j zO7mtW$L#KN)qi}kO!t3KQVM-XnaCU332rFVX(maEf^Si)e=$V_YgC7_7T52n8a<81 z7z=wx>GsWg;y=YJ+Av?L=ipz(2M_w6DJO27f{7xvN)=}W@NyngVkP< z1r$;1=B(>@|JWpf<*d2PxP~+qS;VKUVDXmAWyZ`E-mmIid)Q@0bjf9v#tUuStf^S{ zK7qb;WhqZDELoSVIBp5M+Ki`&%r~2POR5fCd!s)&^Vf((ME&8gvbFtHJ1=QFC&oc+ z84bosu9)1C4*p^)lSMyBJd*Hk1Ngv-I?OAH-6hI-@A{*)2-8-5EVXp?K0!QDa0Rc^ zK^Hi#|2*#H(~W#0j!s*QW-J|?F0Hr)m+{&0&*inKL*IacB{5m?j*!n_${|e!yM7F~ zHVO+Y%$xXGzUc81?R6`Akd4p5E8GxqfM!x^2&K&U*?oPn*Giile;c(O{}IwcqF}Xv z3cfJ~;ECx0IU`M-Cpb+;#oJF;j8Bot!`1=UUoIJhrO%LFCwhS!Y@j;*Rhu3p_x2Q` zHTb`HOQ%i;`5$qs1soe}(@n6O(Xc3A3Pp6GKstCFt-JnV>N&a&f2R$mFYVcBF$L zYE(E)HpWwYOK+bH#LahdO*h%@>8l)$X~-jX?r)Ym0-lLs20*RtF9NGhU^z=z07fOA zG_nw%pWFfQ3=iTb@V-DSxK2SNaO%I7J8|+itc5~G!)H*B?C8>TMg3X6%3BVL9*3w+ z6r4gEl|u(YqhCl0CLyOp!C>CipWVY3d%dcGZ9!uENvah1wM9>nUw0h*c`fVG(1C|W zzlCbbb34x!^=BXIi@nK02xBJ(o<$o}kFDlUv`E!@#;e5-8bVB6^O0=L*H8|Ph6s&4 zEshPMJpqY{Pbcgpzn$Z=FL@pVu8ciyPssOl@Tc(S{M~YIjNNWtvW=f1Irc`gOCSi%PEYZ!{_I0E&7wD`=J~~s=xO{W3zBCg zpU)jSo*%9{&PuoN;&EP#sGm0&Eo?JgN-rCo=+}=Xgk;YO$-bj2{ybCBXm|v%s4LIw zsQgMsG;pkn%FC2sr*ezA)A5L2O?p^-*ymS4mgrC!@L^lI$Hb5CNJoRi>gnPQ{VSdH zp*x02B5I{!<1uN5BEyzbjy#n3*@iE+yoxlXK7@{qh6NzO4Rid+1(3|BdOjWe*M-8T z?0_Bi4d^ev%!ch0!-Qu#uE(*P4z83UB7y4B1t_zWXj0x?e<&nI+< zlaY95F{SuL8KhimaU@YW41Eave=QWRh^KqReh4}nL`t~9-!7C(!SB>%*uz}Vq#se2 zg8xyM;9umMQeBrx&%LwZ~S~Ae(P5M)eIG=pz^O? z%|Ht+!v1f zG8rmqRmpb*oD%38q;NuHv03^Y!SFlDmNOdfmqyv^g%d4AjIo%cV^L>GI!-rtaJJF- z9W*I8HFYu?RF_Y`=T&Mr-|5%rD|`7y9(s0hY8Ee%C>`>|XnX`$eCkCN2qN8_s!*fdQR#a>~>?&dFPcKkxt)wGAG@O#|neMO?1TAvL&+#Gokjj<4;h1Jw zBYW#Layf;1Yn0AnjZ6ps03fATfds2X@=6+Alsa1Ga_8Qki*l1HhAONI2)u7XC^&He zq>WCS$^tKKBR+xohb4*+Gm(p53xhr`T2%|=9`#)zkgiV7xha>a5;PL7q$OF zHkPNCKQ1{LvSYz{BRNJsq~-uA9lTP5>Zd^sNWBZiSOsSREtTFuQiuRV3Z2Su3DCn- z-6loX)fu>aKaB5eW=eiCKw#g=ckBQS`v?tN_Q4GTEc=F1n-yIg|+-V65t}h2d)11L-xdp??pkfbSe0XDTNH@dR8mtU!h$%2b&U#xr9$)=)-wvL2VX3SN|0`fB>MFQ zGV9<5mO z*}~-cU^@L?R(V3m7d^Rji{&mpvG^0HDpk|LY2&q$Py=i;j*{T~%ZfjFx|Hz6))B!p z8h$Ih<-AzBwm$4DzvYzZF{N87hCX_tCPlu~OFvUl|D<{WDasqRMXz~^I*i!z=n3o= z7%}wGprXUE98(4z)PthK65I}neuo2xq^hs2d)Wv+SI`q*K zrIpkJs!(%w=#56%4xTFJPx9IlxH_Ht%@K240ufpU4whkz&W6!PyZ4_S`c$OUmgsn9 z$5l6Q?`h_P|1LYC@G~dw$kVFyrgSiVzM2GfxW$M+13hFC+{{%ZU+^TUttLT{+!2`` zCXz4W>>eF|a+tK^izPZ!=NEsJ#n1vZR-*Y!yAY^@kR#lN(~AYsaKSWHsyTm>XPN2hD!Oy%uj;UY=Qi@VH0{q*rByj`KdhqLmyZpUU}^4xZHo;^UB* z2|rvQ4rE!$2+SThhDDCO(T}Q&DI`%ZPYxq5-^3fr@AM>>$mbZJJ$FI_#Dni$|50!q zI?Q2QTG-FU@85a$>e5SO)tFvf;jG8P3!&y0;xi{VfW+pid?2M6gvQYb688U7*0btv z0+_Zi~K_0Q3TP}m~XV%1%)o|RR1DYxn1&FZzSUK@>X!?gHKl}pBG+{hDlA0-_B zFL|eY;oE3DxsZj{;TN`JYC>EK!^LWdf*(@;k59i7p8dr39X8%L)^J}SNU2nuQlzg7?yWgvVhgiRF8{9!Nl zCA}$HZ@VV-KY630+oJT&5e=a9Wv%ooop*4kqVZX!@D3J*cd)1ME})uHcrW8dDZIz4 z{)oam*i(2f;S&n)5KrMfL~x6@h{8KqE4;rIg*T(`zH=kpaSI-VmrQr?L+WyRle!FR z=7QS0L|qD61lNt=KG28iNA-s6!heRMv?cC=(}|nXY}3I z@eav9^xZ{N6MgsD%&SDfbjqcJbH>Q3IQVuRL?;XTqTdrWF2hgLRT zS{O1Kg^Eg1bC#nl9_5Q|A{@=eF6^m7wA&EM44UoqlF2P7j6>ktT!ryW8bDz@H2EM$ zS)nj~hv(GuFti@ces~qr!6gyJ4_Sp#;_)E&`Lrx@D2yoMc^QQx4n zh4IG#QVJtT$UHrZMsZq7bMD~#v*0jNc%}N=2_!LM0U7TAnUZ`3WJ}p~G8o#MF*yE+ zkiFm)@!jUi&C#?@tH{k+6VVh4MP8o1doG2QGvhPvQcoaF(!mwBl3@L{5_LGr(;s+G z={70z%N&U_heo6)*W#;be<4?hC4v|6O$6CE`SuSmx@D>i?V|v1_1ALG!1+>r739e} zfo9J!>xd6`^`{}8ie3NrfP=K@i)L^=x0vdXW2ek^bplqxf4t(fHqzDD%AaPRH9umV ze0W+_nf_dD==cXGDR0$StufCq0M_+2T$y20<49qY31(t&gBskHXowkArizcEIEB_JA)rqooN^IA zr0$+Rm+z%7vjob~t?J}qjx*hr4zjzQS;KGSIiH&D><|TFuRmT4a`=Tt0ZgABb1JT! z4(x7INg1bbm4ENfI&0mNJP;y69uqO;mXWzH_`*W|ToY#fQ_1(}{k_$Qi=?7W*ZsN8peMOyd{ZHsZ(09>DA>@CnXGP!p zxMlrNuYaar3w?{{bWh*8s`xDwr%;|0>P+8<`Q9yk=THUu)@b^&>1|u=wZ}`qE*(5g zah^K<=q$jsivHtE?ylEt%-%0&VxV0_5WE4b*67M-#h3V=-}ViVJzr7xf}Yfm(kMv`~56^*&53# zFXaC#H5D80o5&k4P(+@?1$kH5mB>@G?_2%na<7Q|h1QBPA_aNQro0lBZ;;(;$uH%~ zo+r9%OAGYjP_+lhFu^7bLI6baZXKpAQq3$e`HW`j@%Vo0Zhdd86}{_NV<% zFHr3-o%O%lzO1*&Un$tZX$rO+A)4^NP2X}99))*t%VmqyZxLVxCNI%-io80afbd&D zstyVW$2}|!P0(YMA;R2oR7>_TPh(yHV)bidZu>V6%A!)%kHYcqJr)_FXP zYC)MStD9amxCLgt45 z47V+dNbjB?7|l>;<+n=8Cy;Jdkb);GNNb}P0co9r)FvlZEiMwHfS(&P0*4p@wrI=_ z*m$B!Dq)z1)t6KShx~J5uI2ogEfqB(JE0Dbtz`%6`+Pdoa%OQ`Q!Gc%#+S~RTp>SZ zuJkkHhoECt$~LbIjkZrvV}Co*PT}og@E)m@iuX<-g+=kJDL=KnRZ`JPDzGJuK+gz2 zIbjUwNs1n`F)TFu0?@OrtoT^PWxIQbJ{}#~&4%yW-ou``)dQG25(<{8D zziGnvQdrTJ-s15ExY`#ehYqk-<~((rf6V^Ty|>p-xA&Wwx~u<7?R7uDrL!Fq{6FxB zGQq#$wR|Q$ecJ2k5b3Fu+>aSfpF7QO642CuinEjKL2;!#m;APF>%MDsFGs@k!e5*W z6D9raO~p+{fE@xCF?6J9>LG^2!eWAm=C2>XMCXSpoePP=<7QAJOU|4OZO44mwC23{ z4t~0Coa$=GYkmmLYkq2u9%}^HDaI~Bajj~Wfvt3N*9BA?Wghp`g9qy!+*+ni(_NJu z3tT2(6QMC^^DeikOlna`Q|sCHWus8ln1+xZIN zsVZjK6ul^iUTQwdtmBjpxW3BAjEuNf>lqnwuAoKP!|h%PB|q_|r7@t7%@?7o%w)?M z4N7br9#^c;=`|3j`NVS@~#tof2o1KN1#!<^Z$^(-Q#nJ#%JvRdwe|kTh?j5%&As1e@w!F zkmu{omu}|cks5yW{{noNr8wC-R`3n(AhMV-aJ#JM@I^7K#`y`tUd7%t`x*6^zmHK* z<}dk0+4*ZQLBxr+#|qkoU$gqb2#v#~9uE7fA7tz2>H32XwtmYPMT7MOHZu;A{n1Yp z{TArDi@dsDsSlwJjn`EscvXUs?hPdn+M<23+K|>>I_X1M`m-tofC2gY0AmmRWiQlU zlv}?oqUtvjp4|WXB3;0LF?C^?I_e6!4hsKj-To;D)P9Elr2o2p>hc4{U-Z#+w(7v$ zRNlvUXGS1gw688E=V3>XJJ|8wD;qv35P>UYraGX48>lI#&WgsCg73<~2{dD(K$2 zK@HUMLDhdm*Zl*+Kzt15bS0P=zA#mO^u<=ef=`;@1-{F63Sk87y+Rs55k zPr3D<*7XAiTmME~e>tJG{cb) z-+zGh(Vx`$KuXIgLCN&oDg3IrXS(R$(n}tv;QK2T{HEw${3)J01@9`UPiW52E62L} zN`J~e+P`b0(8vFzgw>c$|Eui_4`P<(SX`&;)<$}*6=7V8y&&EreCp!m5GzZ&=`y|| z;SGJ?*`Ct|WoP&ss#I@YBX}jfIe7SLJz_a(PymCC3g7|E_e}jtU4PgC*Z1U|ET=wT zupE6%bV{b*{`qrq>Ni!Y`hPl*`r~r#J($+B_hNv8TP6ZZ===Q~d+$`w-iuHmYwv0M zQ_Ma1w!_@xclEC5Z0>1)SUL8cHZ1!ooD1F|{3OS15X1JrsD%9T=pfI8RvRXOtXxSL zuQRct+26IaT-;*gCVKt`bzoJ-f%LZ#3jGB@74(OKzW5u&Ts;Z9S$U158>$C#zGcSY zuoa(ZD@c7A|C#I|&=uig4nI?e72bn^kPdCqW9{y8Bp z{uGa_p5sH$bFVq={pjpZQNp9c-v7Kpb?ts)PSUjlw)fS1@a(&XfW?HW0oDPpXR{O@ z!GQ`79dPbO{}GwhNk8p0pE~>}$Ln7GQ^H#G>fh);$(_#~?cYpIm>rvMJRQ3;0;jiVym-d^#a{1HH_3ICy zekcFq>hG)e%e?vrzs7)!fKlD>Qes{bMC3H=YM z|FVE*{JA%3c*`dqJiLrPa2lXchhu;eb+mi`LPmPzb5uAH2p z8}G*cpk?h3Q{@-v@(=HC`3rRUx%*rGJGy-New5exbT(eXYUPDzJdyrNSTW|7@mi2{ zikEbRXYkw|NPI@gi%u(COgD3b*G%{FVt@44PV(Ydpdv3qKs_jV@%m6zf77`AudnF2 zXa8;k67&0JAc;Ia`2J~q4&^+fdWALQA`fym>vwK^Q^{McTpQw`TlH7dOO@Y#rt0hM`%%6VzdTJn(f6pg-}8$G zl|M4B+Z^b%x!?57<*##e`^(23MEiNV{RZkDkpCjL{in}R{VzM1_T$urr#~BWfbD1K zH$vCH|6uFq>H3!*Z2gwg75;+`wtigKUmrQZ{^!#F9ALnI3xToU{Fhrl&CeN8zx-h9 zKdtNUun(a9%=pEr%lJP^-Gk=8b9DPN4yOG)-G0%5wx7YjrBvbn%IL3w|Bb+a{)>RI zzvJhve;xK~SmEKXRCs(%`}Lz!6+T->eGPp63=CHKyMb{~_H>56mnb;F@e0oV+rw)8 zllAL3UBBPK)_?01HNH!&103HB{&(s6^A50nhyJ7^M&+arXI7e1E1~}C+}16f-@An{nJ;Xy2^=Ol>^sT zN;QDD&ppWT`t)Qq-gg{q{lDn?;}5p}EM33f!PYO;^_Px3!2aj12OpiJ=Kow^?5{lP zqz^r$>z5or{SJQbWDiXT5*+zjmFCE=V=rXCM8@}TAzn`dhC#0b5o8N`p zCHQh&|M@@bbVd^A7TIznALlCZfox_p!eJ+IyT=(rNAWUqEE=b%c6%gRq)_GJ|5Tqb zRH}|cQ)eVeGaUI8O1{;LT~e+7W-BD9IKk>5$mo$hwJ|ANM!r7 zQ)Ed_f8OAe+E=)*{&=aA9D?flAEp9Bm2;4l>Qqz)X))itPC4A}(Wxjevz;R0ySsDi z{&&g`( z>T<@XLe>5XeuGbU4$12{2C71Snkyt{*76(ic6euPKP$07IFl1hXZ2NmNp8BuI~|X6 z&N!xJ^%L@)bC(cP9F8Z4pY`S7WpNE~zNoHp4^`UVC%!N9f^zB2UcMmGM5es_U~k^D zrR?ha2AnC1PJb(Lb}wuDE7oyen9jL7)K4cYEhbwO4}qF44K&KcP@7p?|0|5W=~_ED zv$NB8rN5+ur_-jl9iWf56=jNtT3zx|cyj({KZT?^|5IXGx2Z!x*k=J4w$%;5 zT@en>q)lUS6CEwqu*i;fn)$shDdh&6xwLQ00QUMzXF1R9-dj5HjvTb6&vQ%g!Q-8q zxZQ(yuIA-eorm`O!-aQ7s3y8U&&@jzZ4gBi@4)$-=QcpqfM34qJU6zToc=x83$dBN z(NDh1-pa=i793x(qP0B4sZqY<84sk>4Nb8M2sr{yrouqI1#}oscEUeWo*e&ul`<;9 z+vSo7ehDPD?Eac3cIW;V7dE;ig4fGy@{PG9fBJJqT(I3E9}1vH`TbNp#N56__F>2t zr;neI?Hu!Ov(|1y9K6324*O&YAYWkO;7oBb`uuzpuh z<5rOpmYf4FM^jvH@yiu<_=@~!U(2lunND@S{J7(J9E5#E0EzSeG50QDPE}XKe}K&6B$_A+2wqUCVrlDzVq#GN2WJN7aEzr^tyXRA#eTML z-&)^R3smh4;0&NB;9YA)sCr_wf){R9^8MD@=Omd6Xut1$|NoEY8Iqi{_u6}}z4lsb zuf6tKlrzuAgmUh;2+X>0HEq(7Xl3&*o#3cWFhzCR>28Sl zdyB!6VO$!55diJUE|k@1zkfN#7O2TF*v%T^(-I)@}y&g zrXXz;ad=3tW?c~`{J|(u z+e$JxYh38z0rwuc#Go~W<5F1ip{&ilzQl+vMnv)$uE5Xa3Os2e7=xXT6_d$GbbASe zl4lZV?qh06!nK&i{Ix6(EfM5zAspYLu{E_rPcD=2{A24FAUUbKqQ{g;Ssez2Yo(kf zeuKqwqqg`aByWgUO4XPhuG|)`38gAnKHnr7Np>nN=CChVEaR9E!avM%zQ$1`MD zAX|$WU}_H$HUmR?VS@YLEuUyWU79n5t>H

ti8^1&&T&$UMF8C=HM9#Y`Pj%;Vx5~Sn&&oDm4yShc1M~?0+2eadEPMu*p1lf5zMv>JabN2#F{)JS8b>3 za{I{(_sRB`%DW=Ohy5nZ`Hqv}ys$d*Z;;BKmZ}NyJaS8{^Q{UN16LEGp}^3J1Ci*1 zXy4D zn@^71oEZ6%skym=%NG@c1-QoFjYzx_39fhnn021oMnIag`6jd}>`<3Yb|vu$BE(#0 zz9J)=YKO(`VYL#Ac2QGx6&Jlz10!7oWcSGeJxN0sT zq7b~FK;M$JTeCt;+^;w75^euqO1oqo%|A18{xEc%=rohb+ZW|4z@$*1?FxP=Cr!gq##S#AHl?d$7E63cKKSlF zh%icPnH1R=Cx;BDbNO7maLWGU=)d+xB~*`ub*8aPG}!W1x5cI`10O5kU~;D9T}Obh5h?$P7q-sD#ppR zuJ%C+pys)!h_FCboYK|A*GL{zpl-Orq61+e*c5G0g$4H`5f(w+nCpg3goX4!yPP>I z@_m&H)y^Nku{Wu4d+FPr_PAd116|d68u{T#50ZZGReDUD+WxGki4)GRsDIC|ZX7=> z71Zq5fwps1dW=))kvfRjP*Yv~*yzS4v7!+u^{oU5oZ=T`j4Kf=b6{uz;(M11X^nFQ z%X3Ds3>0~6^tQ{jBGDxJDA3~y3bfQND$wWjR-jF0;v@;T3vUVvv}8ng6=)SKFX|GQ zHWkskS2bTr_cImjKh>?>;TBUO7p2>_`u?cY$}SE3|FZdB#c)#16gTQRIZCH`Voiini+ z`0OK6)|VATO53)~u&?b!h40cV(4Gqa-z!kk!w8bM@HZeVVr@lqqrtlZW%V^FE6vv6 z9K^}9B2HA1cQ@AVN0d}S*5BhJKSrMyp5yBCaG9^FI^TrKpSiLuGV)_p=eq_dV?yL` zAzDfpZGI`cFV*6WI05q`@=v43$472UxZ5RQ19XT0Jw?)J6x8^0!L8#5LD@6;X7*T- zkx^83RNNAk-5-$+B^rC%e=$~B<5wG=(07eLm>0F#FpEH|#%u{AKCVQ3M3EjBQ_qzi z^J+lYqx}91E&mE2_L3f+mj5|=+$s?{6ZU1^BGYxe2hq{r(tUq-pVqyl8zBotb^gi7 z$O}|-ymqXJj?H~UN24n`8oQeKx(CrAT(CrRH1rl7S5{}AV2YIa3d;7*vHI}nJ=rgW zCGOeEQBW)li|-@1KEWuy=iSb0rEfKUSSI+rEfPw&Lz0`2kHm^#(2QVAjMOx>$1w!O zkLr%pGCE&(BqSe%>%-#Yoo9=E8dKV_z`&-5*$p#dM=GU9Po#2|1oDWECt^K~Vs`)u zAt#%9pE1CZ`m2&BY6!&;H}?>cflx`FPc?|Cm5d+tQb!V&W0U!fq4Cm<|0#53W_SfFWLoQMpS3G0Zs zMcf`ub%dFb)N}2t+Sk+$Y2H+J^1#5&8=)8?61Tm@a0qi*%l?+lHzx!>u_gste}&}G z*iYS20-It5-T&1A*jq6!r^)6`7*LUHGHdTqK&UTn4U7|JD{5UyB;6~AMcc2e;d2C^ zA^r~MuP*Xev9PEu1*9?O&S(-+$8Eyyzk>$9KUS0d%S3}VR+Dk!d zJf9KI9}#IkClqNvgAvcKi#%2`VoB-aoc^Xe;$8iYc&?0i?%qaxr90w_0pyPOKTAgZ zHr{nJ*bHWXN1HU;-uYD07DvxUI8cgBaO~QrCN88RA|h z_EG{O{I^W(^#F2*SyLQluADI@Iqx9Ih+1Jw!tKupPg}86RtL+O|9#HXmt@3*!@dHy zhWAp(HLRAjRjlnBQ*tt%_D53{dP8qaOn))`Lys0Sj;6l`%p z9Hz8Jc^7$*i_l8M8duHVut+;cF4j26V;tl$?r{G4A&+A=gEyh_XV1*EKW}kh~N5L5Fgs!r<>V(ix6GT{hd(DJQ(m0 zJ(@#|_?+9HLz`(?IbDo-%G{;MU@0Q9K_rbcyT2kmp`6)Yk)BYF>90snDEstRq$iZO zp6#c*J)x}VuSicQ_Z5ot)K^9R&w}v1^Byd;#9<=)1!jE~95QZ3n7ufMJTIdA&rcwx z8zh5KOzPNUMdntc!{k-^9LK{&;hI<`8LMp$JVLgzP5W>TQr=z03*`IltN7?{+rs~H zBBBqf=^}0%NeE5B2(RdkM#oP%=`}-o;Q{$rz|A^S< zR7dSAhhbeC)nvU-Ca>7Y9bzw!+lMyKJq+cQq%nh#;gSfGVRbd}Oe#h4Du4?8sN)HBtqGw_IbPgL*Aak+ak58 zl_cgIi!%h;rcj`saY28<%eVuh=8TYFchbS%mt*Bs)ep3cP-g)CES~vKK3;^nJ3Kq z3#yjVderJ}vUX?YfY$AvWy#o+Fr^ra(Xdk#uVPq|bCXEEE0XVSYTPoLG|}1&RwZpI zCy8>IX;RI~sl0L0;(S)e&~~z{E&G51;k2xrATh8C@dAZ~&GHs6D^(URJ?|DxSJ$Ok zU5h|4XAvP?T@(APt~+_{tuB`R*PXT-DUTw44(IA`{DWl`{NtD4)@8q3lOv$SF_ zv4j)$K9T%cdtP2I$@G*guTWv~zpJZga!6+X9xZjAEW$wR5BMR6w6ZwT@BtO6m!tVH z>-Jep`ZX7EYF#4%)aId@&p_k80ro9CX)ig9O_ldqGRS=D!jg5UyHHWB>-J9A?GzY%W7FUOLEaQg8&2_`>W5ns|u5;NBFta`=8oX zdHxg6t6PM)K!n5)M2_T2b%Ed;8_3V^9&s~SaQ@HTBQERfA3-ELZ}GK0E{IU+0R#CJ zKMD-w$NcEYKoCVNj|+r>JgF(ug^>^@?NZWn8b$&G(LB_lsMXbEZTt7Q#}hD+rbfqb z57s%{L)wCcz&dC{_y?>5uO6@WZNjplg{eLKJ2d#nIc_*ts9pCratx*C<`tZqJB{yf zYW`m1JB3qo-SZo^W{@yibL@Y;4V@||aaW=?t%aQ%7y8%&m52}hZcFg{}o~uXl ztD73v%F+27a&-Q6IXX|9qjTAma&)c|1j8G;yPTX0K0{5J$zpd?$F(T20y)T8e%WFi z7UZVq$nEiwp9m)=d_Q7N%xz+XJe7Nfk<9$Ek=r99cSTx^wapY5ECmLi8sY5xFT#;S zqNb~tY<^9I^vrs49_8fxEi7*4(HQviW`cWFU8|T3Xv+I`U_}r zs4i9F&_ZU={ImLk9|iB0{Hj3PlWsG-M92@%A0Xxd)$<~}$#zP|Z0(;U220ODIk}H# z!qA3G9V*!KLsBWc==(KV@fLP~(pI?r!{*IHn?ET_9T{$ZYe=|x_rST0+Tjy>br=AY z&mG$1v2Mw_#tkde24qkzH;4{72=I)r{@~9wJ~3O$K1q5M${i*cm2R7%@<@8VZA=Xa zzGtAoJ)88665X9m`BQ$(5&G(51LUJ}@mMq1h}mZD?0M6X1T&EYvjjZ}TW7g|C218H z;q;j@!mcrOj7I0wn?QN2IFWg(Elf=T=RMl#f&R0r?bC2uyYn4(4(PH!%LXByxP9S9 z^v+Rcmt)LIs03lhx|Y!)>rbIXP594cXwr}WO-*{?_BkJuE)b~?*<8)%tPqblfC}rz zyvYhzH?@a{@fTE?jx1N1E}l)$OZIVgV{Ws)Ud>iI`}b{j{C`igN3vE(8{@hfMyv~~ zBkdQmf4>lKA~T6*d)!1hL2`IoS2(q=tR8IHba_u0mN_a+pF~V`E!b*)24oNytqGos zR;+k@`U;E9_$;hR?~IkJ&%))-UQE!_!~$(k9wvwa=MHg))8=o zVR%VVctnHzyQXS}1NIsjtSY{^nxiTYH&{(&K>O(?QTvI1W%Wy^-J0JXekbjzcyyB2&gy z>X`szElc~|lL1se>F=|g07!qI)RO@Bp(HnxhyPS+oL1u4hG*VmbS?W+-gv%k<1-bt zkwEmN{PMiN{>ig99QA$}Tw{8v|k0$?@1#ay%9lZ@2Cq$72*2Tqn+M z%Sg>RR^&Cj6{RO#Um~(6BC_93jND~J_S;5ei!LxoOj?CgvTD}3^ZGNKk_}WQbxw^+ zNs)*~Ql6oifX)&L5K4V5hD|kfUH^<>&n#YQeSb~!wTR`FEUv}eagp^vTAt2Oego{j z2J38rO|g92&w)*y1DlE?Yt9H6HgzwVQDgR)zGuJp{R85dsau5E)eRe5ZWOWMcX|Ye z+Mw%89Pg~jb?uXnX`g&-JHE+1obWc0A-KSuTqnGzH}_c)-tnA0e^~Q;JU}ZNUr!&8 z#em(<8jR;IlVFs;-Jem~hsr>WTORA;Fmb?BBZC+dG)Hqvb z8@Y6h$R&i*Q7=exuWLf$B`W*Tz)(yzu5>~&aj@mn1)Opx4FYHUhXvR~yNPKczqn@EU*QJ!q0B2M8M-fK8 zUI?r0r>ZoaQ)yJF6Tl-RM_$&~;ZHa6E@55f2*T!n2^6tq+K@Vg^ICLDF{`2|f(knU9U-dP2`*P2s}8a#|YSMBZ5 z(?Wlc7P6p4yD^Jv4`cB9Us#Ggz?>YF%iF$Ea46SyvgX#)DSwe`J91H|PPV~r{KI1Xr!XZRxK6I@Js*|R zY`Ghhw9sancXDq=3@S+sr8s-XMD$tW50}mqaxQu@P~69;xJEb*kDF zmp5^1LMXS77K&SEaf2pNs~htJ)Q;fH1elz-D~(0vz&_T!9HL-9y^)(c3Sd%gWkhaD z?IQ(+FKCTA{j&WDZ>Vo98^sf6;xZZP#==lb_`iF@v!;N+Ys198qV$ zH`ecK)D^$n5#cQ4!b)ZE_;{}FBmUJ>IF`eH5%v>)|^hRstt zRZMJAv{yoF>tfa#6*!%8HSIJycUtw9Sm2?dGR8slS;iweQ37^r`%hBXdaa~0zs^8d@1!hqpCEpVPGWd zm{^%xp_VDZS--fr3D*QEC+I4{MM{ix)NYgesb>Bj9$1{em$&NtJlmxL7Xi@D&^3i*aaTq50Yg2Qx#hz@?2skCNqi>CwK09;J?q%hl|c za_<0ZzaFDpGsz0ja0Lt?#9WQgQj99?OAMpy-iK4e1(N zgWYIib=R2TWx1d9kD=>cDlUv?VuBtP2B)9GTUm;0;x+>;QaAdPIY(<{a>pQp##Aqn zzR?8R@aXdPG0gC4ZibCH*yk;c^om7eh9@}vwEeZ(TU`>XOP2)k8j34oP>&%knpmTg z{17vd$OuR1RC0&JXrL3;ASbPX)@@LQsxq<6bq5#k+O^O*PKl`%Yk7vkJ4~;BBQ%u$ zV}gTNg_dZ1CN)qxo%@;IbB$Sl7bmZfW*>fDPOh+Rd-jsm1$)WK;<;c4VvWzpO5$RN z%qEm5sT!%Bl%Ta-WzF3m%Un;VSjUy_rhn*%i(SRbJV?JUE1piVH1$v@&?*j)-U(5V z%^BEUr;CK+V2Z$eq{H#;oidixiHjy4ti(r7_hX5?&HbkW?Zpi3-GT!Hr6@31qXN8~ z%gMrnY+b4ykp6Q2H799JR@(p>r=eOFdN-=O&(}WjU|GuWCqu2J{3+0iOF5F}^lEKF z4nW)LRP#~ff@EA)*Y+RC;PvN)G{?c#JKR54WZ@Ex*&MZ$Ti|2 zHJnMvoW>#i@Bs#uvNp0yfYO_@23miiYTHDw=Z3ph=N{`HMPJq48X@$e;)vdIBFWKi zN1*iq@Pw1t@BxuU6QJi%YX7)vu9T-`**V2=OgtEREevJYhl}~ylT_Ztk0-;?=1pZ( zt_drIm{N40^=JSsI^CrNPMZ&tLC9(1&H_Ow`6@G-ho8ryA*~eKA-b|m>2Vd}{Gxe2zfUxc~G(O{(v*BF7f`pQwns@)M0tp!II(Hi<`# z+2`olyJ&hyBV`84S40+ZKoHs1HBkKf#t-MD*@052)6~oV{td0`Kof=CYJ;uGp zIH$xV>idfXK6edAF0Ss{0q zjfA$Nuw=Lh>hjRj02Q~#SH>E<0&TZT-(}T+_P8|#wsNqEX<`~%lZwZ2R{=_(Z4Ky6 zk}P1+_=8nIWA~jQh-7QxsV&-wsx5D>L&;h+kTMY!$uMf!<`mYjOI*5!mr_=mOEx~0 z{-L+jkU!1B%WwtA%v39lGNd`ifgTQsUCd zS2fF<{Os)%GcPI0vO^-jtV^tSLp|b~OceD&_7AvLoLm`bT_O`>`ybow_E~htb*MwA zi^kMEFi^xk>d_#6oE%D3i_YY=S4Rx_t}!)X(a_2RzsDR!*pru1F#+O= zvySv5uCm=9u^jS@ut9CHM1sk&ZgwuXQLPTmpr~as1HUM3-P=~lW;g8m)68RWK7(zd z5{kkamK@>yG{i02@imRn5vl0X8itPBH~;$_|1x3z%VXALrfx&(WN{*`5p6;2L8x!; z9>Xn;jBv*CGPmzia^3r!?fk&qi)45+LetoJ;#bKqT+M@=pXcHxdsHoUX>-IAV-ynjQG6l6b>sw^KN>P zux3?CC8JnJFkPpoqgJ_^b=+Lh)`jqgwF4BqmN{8Ix2+6c zv_gmNMuFMwYRBlq;u*$;1$#(BZn2U&b`8T=r6Z7xm`4aN!ZO-d6MZTlte~s8m8u)Z z;ee=ez}jvcMSdfDuhb-iZ2KT$zercoi5_uiyyp8iLf-$Rth++hOH?$n?NhtMme@W4 zUb>UHt1@t76_fUFKp*cyZ`oBjdxU~kO^;R0SaeJ}pxjS%JUyOeMWQ!Oqho-1R&-J(vZ^wTLmi4ZOOP95jnBegrHbv53y;%bN);WiI{jEz)qOTli@yBjuCzexqk=ikw zZ!?K_Y0gunfxs*N(9XgV+bOo7ZXD4%I1qtj?SzUtxh+sFgz2%PnWri{3ej(3{VACFq4-TiD+wd zUbxCdD@?OgRRMz=PSPHjz!OmyScRts#mUXO*9G>=Rl2~et?rrlliS@XO;}uIj$^`b zr-GXaL&X0=Uv4{~|fOFoIl0DdDa^O0p=PU_LuR`gisOyDD5Qs00p8@_!7O=_1V z@DmNsC^o!R#@SUI=T?JS8J~#~)eJPBYaOeH3W=ZvNsa`9Cw2>D8Rz^=h!ln}8N*Gq zKs6q5m?387g~S7x&U5($u9Fa-X@op!KQ{fV2Zh8@=5829_G->U3dVpWrZHIX0uCXB z1s5Fj4BtH!IQ^5zfu$s}&z|T*c$JDYc99vjF|NTS(YPbfdV&haDMss-fpYtk5f|4% zb`pomds%pEpAh9+k<@G9o=1J-Oehoj!fS-!nBtYg&05jVrXB@hCL{2DV#mQI-=O>@1Kb3v^bN|gHuv{65*?ukIrVF^fKxxEd$(;VAXSjmYp}Ne0{1;z72Yp)X^3kP zddMrhPj-YN{`xCMZW}2x=N7(-4+%#$-53KC!}ZCdhCmd>(3A_P#+lbV~O` z;lS;MzC%|FRtlv@XM?u;N&*Rh(M2!4Qm_<6|%AYNAzgXAkym)jMCKyO8Di@e-^qZ{L^kHVoIff?GNS*ZsdG*1}1vW zSC8YP@A;ft{>^ua<)@kQyD?k$EpOCkZk4}?#?m}`1p)GWMoB)NHF#+)S9DjZ4iafs z9R#L_7n@;lkkj$e)_(-r9%i2U?$3nW_Saq0=6vv9j|sIzeYtGQs(kkjD#-oO)P%_H zCmt9?__pRx_nrGI)ls7D8|6j;)51cDVi`m6)=0%jwk&p{N_6?JI2DjuS2SD}K}N_n z(ktUub#W-2(i&<#5;=}(<^9NMP-~fQd#OLj=iBPxJ8jLw*-e$+HfpVW*S)F>Ye^7 znAfHHg{S`-f--xja=bF#X^J$WSvmwE&3?42|TkoV#5_^+pO~{6L_FE zfrgEErMOp!nJBPuDBW$@Rh~LhbZcj;zhn9%XT&UOE^d?cMOO~Is zCUu)f|MBZgdMY9-AQV1cHCFg6E5JQ_?MNqj%+ZNetE|J@6ZTf%Wf9#g+WmYM(VsbEH;bqQvQl`Jhaf0C_j{r3e+J5)cTtc%2DV({ z-+xpMa z!5--8i<0)Ud30)&pzrYGH^f>R@ha?rs;}=}4yOFgyY`*?2G`hVAaxQ_1iCGA#<98~ zBm6Xo3PuqD{8yo+1(YjQbRDzw!{;RCG80D9u-9)LnU?!S8X3D-sz%+vbm zedOvw@K1lH5}{vyGRN(?SS6aEvB?opVc62YfDt4@P@~b(YHHBvwdcU-zJU2BA3RJd zFPL8^k5a)1k+kJolHgCW7UcwH#ZtiueLRLfI&Y7H;9gSr#rX1UBE&coJRV>T8!&zS zcsU#K!LrR$a0%;3ybJ^$G3FswGc6_i!(p}d7`Jydrgty^9iz@q@Y|2DNAEc~m!aSH z!0-K(0l(*L>JPu`Kr9nA19N`KM14N|P6F`W@yh|`)4e98`(fq+Fo!s^6IrR83u0K&FjIU&=GMGMix zp*e@!{c9cH)f$uQDQwgZjslHTk#_Ip?c(gUDmM&axa+OY`m_CCWiW;O26XnQ(%-JX zkEVjgH{nS+NIqUI8TsISG;&$s{nh++d=LwKIUxTv)rLL@T)66kz`0fR5XJ|Iz%iY< zff{aweLtDlKVUh?lF0{GWy^J@-1Py0D?zJsibv1PuPiyE=CbOZA!j&iFPXUt)XO^k zBQz+dQse6*_En>ZC#VlV!6Lro%woQQujndG)+YQFiTBD+K`YhMd?iLfGGdQEp{-;7 z`bgtdqf$?bxh|Vu8_AzpAF-}FAs;+NioKV{MU;qzOZlObHi|Dr(Ir+yaaKTL_xngta z?|ggoADC4C<}aC4GyUby*(|38-Tp@@qG}m@LBzQ`Ns?veey#l(`B_wc2%kVqXfZ!Os^@2?Cq?IT&g1-CO5C{=W2Ok!@5-ZFc$ED~b-Mmn8Ioml!)j}}$>o+HoF<#Du_2+^q&2ZFAAAQ)igL4m%qo|~ zkIywQ!(w>M2k)1cncz=3OtLGTAJbd!oySPs^_so=F4;>I^!@B#dj2lxL;2uYzh`A| zL};AyB^w0tR`X%A9m2l-0+S~=tjPzvh(JcyzS3(3?SXQ@Zk6Lq8SCZ_H1>QqeFwn30i>2&EQW!h zm>dFiTLMF|qy_4Bicjd#&q_tiaVN!&{(v~<0H?|Z{$3SvHK}%jeEc1VUkm{e*|iDT zwfVBZSN?VNm-+v#*@Kpf|3=PFxnnqvSpx(C8OsN|8{a3XcFO_}?r~en%8sdU-mDW1T= za~ABwV9~M6P4Nb1EA;+Q*D&AH$(9qh{s< ztw{#O>}R9Gh8N3t!i=Z57{WcrqGGgkK4KCTg&`2RgC*8~EaeM1;VZ~Ry{Zjw^b(dx zZ85SLLa$EPz66Inqh$yL&g75jC$z;V4MJp{Ss5`DZ2GU$#T&@ezvTf4Q&=wQdtrFr zaEEui&Rt=>yAdUhLc?=A_CK@KFQrd?Mvk>fEd>9+FZI+h{Vnr+@YnyVu*3;pWZ%yO=GWqy(TtfipT?$-O&h&a4deL6s@>X2Eyi8X zOZCOuWntL8vu}&^*`bq?3Q_bEWB>3pP3%(n;8$sZ)q4aVW*>tYs-R!+Vjo@Za8qd%PBUn2J zv~|q=hW6Ryw2~IcBR4o#71P#GBg?zWe6-rov2Z}cYAzVu+8Jq2)E<>xkKBg=LpA7s z#5)EbrnadeqlkmNkX-Ev_83$?__bfMqvldxAE;7}`Ms9;+MLn*n|Q`pAo#fK-7*4n7hdw)8`{~~+Lo|w0_ zDQS`V=>w!Jom`zd9oKvGvL-WRRcnmM3-) z7r=+gCyUQ#4??`RZQxsB46-E+wEdMwF%7;96*YOFHwkx6Viahv!1d}VAp#-LEqpSU%Kjn7slkvjvpt_51}1++l<0!pYe z3!toMYq|5aqxD-pI9f0&`&XsFg!L^*@}YpN4{oHMS?xTAr*aHC5r-2vePliIXP^3W z;UrjuKDiH)Ih^i?V#ET=Okr)04+&9Hg^O`w)Nv@1Z2V{7rgtC}7S_}rHXRzz2cMwN z5)CvHY~)(A^EDnSlE@b-Z_pkD*@3nn03n;SrlOcnmGcWYAcKCU$v$-{N-631b_fn5jos;YBbh0Aj`qHP6nZZBSc_jUvRy<2$sdf)c!;!<_33Sb`~e* zYJkC{FO?5H-jF(?fQbTX_}m~~xO`5|KcqmmX_49HP8Nnf({+fITfCoqj8&t759ZaVIG926|GqRh6frz1rcq({ zWx(lun5oQ+*!j~|IA5MC2$j+Fl@d;s#ZGs{UPZ*Fd;5rpWOY$OoZC}E7>l~etx%8; zN0rD2$>*lxVN{8D;NTkRuwM?RYW=!L;UM`BdI^V;3f#`xqiFa7!}uJc;ZhnvG<>`h z>sWRQKYH!YJbM>@0x*=9i*);!dz5Z}Cm$+)LpV{XKMVFp|7*8m5wgk%C0eRE#p_gS z8Y4R%)-pC0bYMWi!r=Jy$x z5x5$7MKuR$M1ErXxQz%gurp$Djjck-qSO9b7NelbKSi*0P56AOQ#1?cn(|s8e_si>(Tu;1lf}qaE%h#S! zLv#^(kOqN>NMcIqA_mtQqgC8c2($a0kaR}6;fxhjv`D1Y;yeA>bN>VD`}R+SlH&EE zqw>XnuK8W<{f_JJedK2t6Ej*cb|;TaUH_N04I3Idno!hr8sx^FyQ|hMHR=g8Bg$(X zf)<3-olY`q`!MDI)=R2VtVqA#lsP>gY1vu!{i9FMOsZhe1LFA(39vxOn&p+H5}@lY z?7N=~NL`palz7$$$Lxzd{$Eg7O@Q54zAL_9Wi$P;$s*bK;p`hFE~(kCDK8Al8ZHWAw|;MUF0rtlbq9p^gsf)OEm$BdXN}tIym_On zX>;HH6?8bWzK}+;zFvm`U2uzJt=>rv4ard;LMx7il%*-_QWj{cJow6t629{Ck6pgP@WZbCI%c1E+6oj1Zl#Hc?a5av*D7DxhmXAA4E{tm zI^t>b2UjtoFo$>Sr*s76JPlh}8C$TTnL$MCYiRyTwC=Y8H_F{K!8zmapS#GUuQQy5 z+kWpCIf%zy-W1iOKkia2s&Ccb(e~ds%ssD|7n1XetNF=>UT6|^&^Y{ZzK7BMUd|Z; z)m6XNkKlv)amW(+7>b4&7|IqcFch15U?~2+acg5d!%`Br-(==Yt|wM~N>MR&*Nb8` zCuW5^xWT45*v%PSGBXA0o3JjKxd`%`y3x7kE1H@8Aa1M5_RNIVa2;dW@k5RRZdL?8 z5cKXzz%=YctpV^s^tBpglQk#FRlj~xp|@^37;fUO6M)U%IhcnoO$X~b>E3SmIpxE>hGoI#f1Id|~8Xv5g5 z5!0P^IfU;AC(+f)0#UiFvsTYupm>vxEM zdFlB3tv_y$^_zRuAJuRDfj#OAeT%Boq;=PdYl2v#5}9ZvK^&2|%VBmhf9xx7lkSP} zC?rQ)hHb+d%Q(7@*=NZGpBRBsc*VSX?^WLe{y*>zhw&QtHSBh!#4A2R;Cv;@XW*{KI7QZ2u!a9-)Pm-@{1jxgd_wK z|KdXfiekuX|0D4h-|KjP0U*YE$!E6z8wKT!_mV#CzYLEkpvh}L^_lG-VcOsCGuuC^ zNBf&^?LGeY`fLBAzkcTU0mS&rKePScD5%2td$up}HY6t79Jby~fuZ-*PUp*IQBEuM zS6F^E1zxzPb^_n<2)ICjJz6U-v_wpw2P;r+W0{-vIddA}7(13Zw(BM#4zie7M>$Vf zG9vUos$Ec92XpV>XjM)&rw4`-%TCTCGyYk^u&psT)-3;_ybbQBfW-orIq|`T0`9BM z0vA+RLB#--R!}$BE`}?+&@hzxzhR(pbeb~IK3yl>L~78vhZNdTPJ;_=mO`5Yv?E<; z8;j890Ik|Uo0$)O$v}JGg*GdZ`Qjgemaqru3aWz&W;WJIRa%iJ`f8Ot$OP|otIBgg zzokPgb0&BYw(#6Q!0x3G=lz(#=lc2(z>yRU-{r91NAkx zuL4D&9^^vZ$Zv(}@CB%T7wVQGRCxo`7Y$U?*WDTI>v$KcK#}#^*(NPNT3bzRg<2zE zs{IdgGOVNkPjN{>-T?JK4OB<*`hbD@Jr@H4g(AytG<0^U3q0%rj~U=+yTGf9z~v1! z=NjOK6izq5pL@!sGbsWhh5%K&I-LiG{%7f{M{DJYzM3fyGQr_)zvMa4_iu6Ozts$S zu~(dJoYFsM$uj6rv0w6rnp=d-tQ*@6{ogFbf+L5?rBH?4OgA7`uQ4D6gn%3g$fiKG5e2BXA{xZcP%c#YCK`W;1@GR;Q}SX2hTdG; z%-5Ek2Lx`qtx^%-{APDP@aNXkLJ}Az(>mbG=ag~amB^fU7^d<>X7uz06)Iv@XP4o? z%JV_yd93h!hk5?~5-nd}cpfv)U*vf#bI6uv7Y;B7QzrW?K0kn4VqUK=dHo*Wvs29L zyY%(37kjUP^lQEYq+3e8%9#K_t~RAkE-BS)O8s0)U1UlnO{oJ*N}Xp)T`#3JS17C_ zO{pz6xc#NY6HKYeQtDf#RGBHYrliymQ|c>HDhHP7*Q>Z^BfNf9Qp(|b_A2xGQS-X3 zRFaIK)!BD z4Jau!PF|m9ULR~;Z=UZFfh=mO4dA&b9;>r*e942YI97wHZNGM(=SK7JZ#>jedW!+_ zONiPKjII(=Q!U|;C#V3OZ<6YkM8ie?Ezs4A+?S<`dzRD=Yi-no;CQ!om}&>mRj=A` zs!7S|T5={Fs!CxS(d_COGpV#p{R>2E87#^~COFxJIGw55IZq%S#&9x%aU9Kl@0DHXwi~90#yMjGM!Ls7?{UPrwc{g8A~t$Z9RqbUC!;d$TZ&NS4NzwosFnQ62mjWlh?wCbLZHN~;m*%os6J{N ziV!f>{)`K?L|9hx2B=3Hs79MvXrKmMr~*Zx#$Bi(4{Du(dXfvZL|Dojpo*^scr|^k zH&Cy2F(6Q6U4J;&U|@m=b*h2-#?K7~WUJ)Pmb?M#>v%l?RaQ(s_=-sIOz@X(Uj+(9 z-o4h)`W-ItW)C=vN)XZN0-sO>E^ny$djs5%!aM_fpxb4EB8UhBRndAM56lYs1K)(8 z<%zy}K+0x zVht$kqlggG11?k_wav5(m}>tWF4S2?sFi#L>OX{db@Ozq0r^R%0VyB^F3S0bNIxj|NM*ex`8-_t+ipEG>`rZ0dsIN zy%iu!d^{1u4>r#VY@SUOA^ESG93d647AKs38zoBtnlt*lHeyS(Hp`AWoUqT{!d)1_ zg?+O&93e!#P%Fd44(g1$5JR?$PxasCe8vCa3*|Zp9i`fCn`%Q#(C3&%0!=im z4*bFzXkA!gol$9>Gq|B+dsA(#uTqeM*NcP`)VBT;DCt3jR{JK_YmOrDWe_Xy%!K6NU^3{0a|NRjTv@y1RzEiKEpo+wx& zjfCH%dVS%FPJUza=!YmSsbou^uW(85cHTjPFQS*o)aZqhec=RvigI(pniFyw|4ukb zjtdUaD*51lxMefJDlNFh8BPiJJ^OpicM;zif4oQg<^8u``~aE1n0= z6bOCJ1HX2TptryCz*)WO*Y#U}+vGjfukTggxwgOdAK7F5Ewb4zInR5b-}=|=vHl-= z)t}jK{jcq@{;Xd0>-w#~ZPFgc->beer{D4SsBinevJ9g(2 zm%JP&q2GA>p5Y^=_PrPdo9VufP{HYTq;T7(g~PzI?+7Tv1z=3_m`}THoPiuC?5&@bVaoMm56OBM2@~$OGg@? zyy5vA+S0qDBIJ*-{hP$XDk?9cG}bbD!xq#Yaot4SGvct50_n3vy#gJo{SvS1lJ#)> zn{MBom)0(8!biIBbwTJhyT!pyL$sZ; zbX0?BKc9PBe2(hnMoLI-JJmT!+UHj6CSqZs-1uUb1YiBLs1E+yZZkt>tTOL965Ibc z<)q(jQ&3BL!7}$7i5{`w-TMSors^KC@mrgW=L0h^9b0lyg}akN-DA`@0=8W7W88>; zFhsN;LXUw@`ftKG<%6^Mt}+|UDT%Mwj%xPGCyl935W!t?xIkJnW^mvfB ztoJqV3eSGWs*+d+R@J-w-)n)WFQuY zv{_u(#J)FECK}!B*yDb*n>5W_=1)=!|J-eo^3iHNQOKQMq=&@1TTc{PlWjmQXmy&~ zW6y4ik#^L&O<$4ty8I>TzCfFY(x!FKiL$IBjqB$AN}SDRiTC78neKXBp1U5F39f3q zgP~-CF+QB`t@(V*=#9K6n?C_^rrX-vCa=4h6I*YE9x_#jm|92h5wTtxXW#ra>8i63 z0QnW%aj9-R>nG^y@bpb1Pc*oU^>aGP zjt2aSgNZ}TwM%gNnlBy=;!2`^3tl{*wb0j8snGHSHxV?(9FbUYf+KE{AkTzpV>~S+ zRj!KVaS(k;;7GV0J9wbf#Fz4IagpDU+gS8>;eM{V)YP$;va7zvdcvtQADlK@2w?^3 zdOq}BO>F%Wh*h~Qb)@w~NMJmI*K+;koE^>%ytuqTBuShQaf84Wl~578*#G4x z)JZ?&yyN^8OQowM#8dxar$d5NS}St*mE32Bf7IDkzh@ni_v(9eg0YgyVdL(Y9hOV? zbt{~^MseAAi@TpvNTABYZz0eGV*1)ucGaZ{=@x;+{YlA{*eY|?^5*V{JxJ~qjOP<( zR2A8NQ!{UT@Ck7Z^Z)TaL0Lz-qM;*4q&eajhZLV|$YXZfVhW1yl09LS^YRzy&)UEv z(bFCv<16Hq+lT)i7$8{@Sskf;JcrcVrH1v6Q#qUp?aum>WcW{HuLdUte-Swj5*!db zMTf>MEpEeNV(4^qitVafSxWifqiog$H*{*_Ziz(>1Cc-@c5Ic?{<4fv{OE-Mze6** z*S!5q5&nw_`+Llkf56X}^Oi0{RT9Qw9sl*9DnZi)cGYlgc&s$6o{6bbh#k3`0Ae3i z#2crqBgSk?cZUqoZ93b0o-k28Zv%Zh64o+dD&FfA&N{AD04nz=y-593(VedvD~$y-N0!U2Mz)tq%b--tra!)3<_MP@g32TI)-ST!|XOEi4&9 zf-g)*dhqrI&D79|CLdvNjkdhOooKSM9)$fXTK%3H_`7 z%4J!*>bLL8JU@430o^AFy1T}HR1s^OxK1*=ujr6fCSFO|;JF1<;jSrU-Swz1G@Qb7+C}k`Mmu3bQzlRK}qT1E=VX(qTMoH!?*&=aLt+;}Xwh zg)bo`@aMkuKj=ga?^jZ-oyB`{~FvLeE;tGl5YP1fVtQE`nM_l zd$nLlsO=<75&Ug0Myi7T?teAbslX=9rk4IyVlGHhamkkAR+gS?tqFj}eXMDVL{U4*A^ScGatn z4A@@)`7H&xSU@%f9^92^oL8CR*1fs?5^$QxT8cloT*M{9KS9zZ47Wxf5}(;+!e&^8 z&SR69H>>l^OvR~kQ6uVej{bQWl4)L{bWf3+ly0__@SB3+!L$qd=5<4K(ii4Uh!EycNB>>Gw|OJ^?`E zLSAfy{0?UoLQ~}5=E8uvhUR`lre}o3JHPvq&nHQdzARI16cw zJ)RmGm~?IKb%w(Ym}reS0m9O1a4+Zj4d(%>k+?mjUxY}g`rbdvIq~>A*Mq>2bXI%j z_LB618gt0p_zkokxc6>7_N_kMdL4*`Zdvav!IJ2^X+X=4^7M#3^fG(4_C*n`%1vxR zr(nEHuE|Gymub=OsBMCFv@$LoEiu{v}7IebI z{~imqguD1Pp9Y59Fuqfs4)SFe7oL6=N>dsaau2h7Cl>BfL_<3e%=Y;V)t)hu+z zrBK!UUCJ%RWj0IDD`U?s#yfmEY5!h0A{h!&BQ$v0bWq0^XAbuj;@u17rTpUx#~k^T zg{Pd>6)?^65T4X>?Hpg=#~rN5CSQ;ID#?1Jt4XdRWn@>Q`HTqn;2e4p*Y)(GLs+j- z>2+8;S??UpP`LVYG*n%`y;<8}sig)v7vm9>GbvL33~lMGd6 zR6hYp$f{l-0|K%EDzc&v{e`@t#BcF(>e;v?FWY0| z=bOey_i3EFd%oNk*kkU~E^IG2kPm0H+eoxeBOS6L=7wFKfYy|-oI(N&T9NHG#R<_#_b#pPXQR(r+S2x16Z3T zM69)^*?;4O2DJ5 z>58_c1CLwpX+!ADZa9wTm9@q*kB2A-Yn4>;KF~HD3YuFx)nl^1<|(&T+`M@)NnDsL z*SvYtF7Az@d73>BhG4jM=5bl`9@p**IuGdnlT{7q4LZ=efx-pWUEq8lyPWSmK9lw> zHMEz^EUcCf^7fq+Z@G3RNg=P@k{S_j{4_;k4sLKGS%<_M*GiV&xco|g;L7PFzp0ic zwpW$&n&`E+)B#CMh*PDOM2kq;*N`ai<3QWv(7Id{WCmn9B&3ELRmS7W=em76i_ci& z&eUB5F_;590Fw5vacBDd+`@!?hcuDM+#<7(Y+NsWzHVk&GBbugCmUBvV5vFuGB17p z7ZHR^%D|F7pG}|FOP@!dK%ZmQxxGaj`8_;E`66W7&7ViFqi-t9&pkUzein`kY6IUr5p! zVbw!K$0fTH=aa>vzve4zjVr4mcx>+^P5IzL-opaO(j@8_?d!!=n!BR#!ks7g{pCnM z?H_l0Jsz(8OB5T8%U9FC1GRtMlDmigY4$1Z@fM2?0UU05C4Kl%e)%I$jb-g^xRDGE7 z&dT=-3?=klrfT^473k3K#jO2EMF@*{j~E@2QFuN3>@8v;zbud%a9QBRj+Qq9&pLre zi3cdR_R7~gTzsiLuNZNaND@z(DZ9jl zC)&(0Le-fQH{R@_1cwnheXomSoCY;q9U*Qb5s1=ANUSLT4QZoLf9PYy;`8CK1D*Q9+ zoWb%1pFLZ;h2u@P+!;Gel@T$Vg)Yi(>H$wkJFNUeHg%-O=3XosKR3Qr+w#FV^Tgwt z#?qA4NTTv2I_C$70`@`jlpDU)tV5cKVv!ep%mt||foKHhQHqvT(FpSmV@l>MjOt^m z+50*}XA_-cK{F$hYoH<3lLNo6x%=fg6)i_}P*1KsH;FriJAShPo)ueiICUy~zUIUw zgg1F6nQ4dUHpaJSA1f8C4IQdM^-*yy6>&eoRV}6xd(V~lFGw^UYYX-cljcjc98DAa zlk<9uaN;_Xs>RjmQIi}L4T1uRq?;g;ZmK)(KwDZDWYu3^C{=X`nJ>D|hHI|SR$QYR-kC`tui~{%GP8CaV|Za_ys+5n2_im`UDT z(HnB06~iC2z|81=+GrT)BP9fqw0|txPdxKi(S8z*FQ&d8&z!a;-uPSDMvMOQl;}Tt z#Z+{O=TcY3?6c7d@IJZcInID$o~H1zMB3BWC^PJ{gK=UX5$gTXL-l+tx-hf02nG2eb71ds~u? zOTB(?lYVoYVDz@9MF;Auw@LD5`dw8o-d&b$bRhBKBfOE>AD5Z^ajbD`3L1FabmJPj zv6XIk3F4aR9Ad_Q)b}}BvwtXs?AFCg$Ttql@a&&%zJ-wNXzVd4fZwWkGqvy%d7==yT2y)H+Gh=<}q_(_A=>!^o3!`*Dn8`spiXGBfCvx z+Y$15kEOF`QVg`p;SprCo|Jm=%mP(}t#%oH+#(Y#clAMm*stRDf?0eO!`1w_y+lfA z4-4iY<%ni?$r^Tps|H%{qoAyQd@+?J5^)8tfZP^28WY1g48~jFFvl1<5Lw{!Cih<8 zVLh^EfzuNqT5|tY$1JA!hx*H!P~**;F~89+W3ZII(QFAFz$riM3nf^4qxm;6gwcyV zLQEP;h!-L1^76ofi(kP8az87vkAg?e-e#o+F-g!~S$sizvvdW@fQrLDgkp84AYup-9GT?uP2LC9pmu1bHH%U!nk|F*v8?=-AM*2CWX zwU#plXwfUq+`IpjI^4PnH6dG{5k(kV?w@ehO`x!3rc?%o4v1w=uf%V`KwJ+*)uS2u zchL~euj?BB@9ERq7l{(Ll4u%eSI{v^M4P<6ii^ZpKK2+L1 zN*s!%+jR2{HR5O>nw{2BO_TcsG${(_uSU}`8-??3`I7N})GV{lj$LBeKZwVUD8*jo z>@^f}_fy^hAzrcmzf#F~ygn&zMP`p6bMI~ut`ayh=NbP~Nl|vsHyZ?s>IUK8*;9Sk zUb;be_+)Q`z@ADs2#eN%lDFnz$SF)#(kTxcQ4qG}2rt`Enk&@DvT@*R6 zK6|(G^i8F^oyCoUBpD3Vb+_|1CM029!$Nsa%p7c@c6}bO-&e5S`unZRws!etrzL&1NcNUtVm7*Q;o4p z618*UnTcCGp|MVc29`!oXf$#(XoQAKbV&lnCK_K)uZh|IyVr;^d81LvSZGqqRbHoJ z@r^aEORYxAun|NdZCr*(tp6K)$#o6HJru@mBhfP>+f>0U z3$$)QJ4Cx1N(-)b*Z*D;?NaxWX2Yf%NbbP0Z7M2G39(&oEEtA$SE+{KrbHh+dWK=$ zx5bU^90F!xuxWr-%D!Q`DnhYKTD8l`#=*Wo+d^3hnX2=+nYx6p=GZM_pKUgo|C_xx z0gtLY8~ziL$gso-NHi$cQPMUFYBcRz>`2YP44eZqEGkOthD9y;d}YOTJl*0+mG>w*fBup|LQ1Bf7KjaxlqN=2;_H|GDlpXbbE5@Hv>_y2v@cU>Qs zX3qBP_j5n@e*akbOGF1MOgrRkme=FJ6kY^IZ$?OmqpuB#SPD98g8jQsHK)1HU%yGF zhD3+KgTiSl_K;Y~RO#_F?1*-c)1)dzylG-hi`q>#fGMK(6?Dy;FWq!Zi}icX*H$&< zrn6eb%g7Uqk5b#cI?HrEK)-s=z?25bcc;p)S3>bFrgM#EI(4!^u*w`&^{i~w<*KXc z96zhX`HI|K(X(oQol8FyPur90L|L*?n#6*iVq`UQwOT~e znY`Z!jyyxi%;&nXpD)ILd?@=VtmUVIG0LxOmK|_Qr6Qeeyt2CZPi*yK!MFVN4@1m! z7X#$2P?iN%XhuO!uEX%1amTPm`m;>v2kD;(yJFW%V^Qb#g7nPb#`t%lGw;$?lFowt z6N2A3PDb&Wi~^EGGo#NQ%8Uvgq_m8pRQYpSj>daY3_gV!wc&+PB%}#EuG9i8ru7Z5 zr&bAk(6LJL75u~fI~9-2|KIj+ozWA>kP3a~Nre0jQz!r#`ycKM%3368|PWZs~blpuA?S_@Xm zpN_+9^+4PyK{vFrC$y@X3o)9<0-peX7;}%Qa;wiO6#E5dB8)s5Ac=~r!sL!cbyoO&d?O4(t?&j~@SWukke_GD509w)ScxN) z^%d-egGrLx*o=|;VOB{S3+mQv$YdHyUXaV(>*R7XmsHt9p3d;AIJ7U9?+P9w&Uop~ zTxQBQg!>e&iFmE4>UzJ>Y zm$VtbG;@=XN{R205s0|Q~l)#-D#LSXfctK@5JNsIw4lG6oZc6(eWACL@? zcIo{TCas_`0He;|jD;tYK;iBKG(yM7zvh2@{DcE-srJH-=esT+$ftKRmg-1JdLD&v zMpyZohZuMF-wWzuvewpfxZt)wsMRNc~#-nJFIK$i4NFd_I_`kvC?gJ{WF#XqT zG#zIF*J|pnF#OlFUg00o>L1iUlxgGB0cK0Ryq145h2-7}iGg^{cHP@sYZ{6Bc@y%w zF8kQ08pQ>eIBjDXr+tH}T{r^6UXhfR7_iPesWP!=<)^aV%s~5$iQU$-$bTLeSutkcIjcX zNv-C9HuA7Z%L2cQv^F==msFiQa{Qqj_ITK$yor}7YFRWK3Idww%hU-ts&7xdUfA)D z1U&9|bEUMO9R1t10>b6M>oDx&*iS?GardtU3Qy)?U;S{PJlxAe9({une7A->q*tF{ z>EssvjyI{%g{$>;a7tY<9dpXl^727m1S5wdYPQ>@{b_;x za-lv2X`y{nALBOE$c5QbSD9UVGxa9=s^9Q5lw4h@_J+UiPPxr!rxAa>!<}8a2Dj`? z`P;Qh%OzeG)y%lR?gn0}>F(Mp=70@kc8CXLMJ~!)il0z}S_-^3uBRU4Tx^v^cYq&H z77@cGW(wD+g7`RX(}NR?Y_Nzb)fgSnjtC)BK8}Kb2#`UhU)C{P>20q&^KQ~V%#a39 zy@c#O&=!*Y_XyUgEZA>YFZ{ord&=|wTEYN?u>eF#h=_eKFTgO83-LEx=E(1CF2q}T z__S=C6s}*y;@^$DFQ;fTiuQ^u`Lwgd_3PhN6NQPBRiY^6`X$XE6s~f=h29abuP~p8 z0G}L~TrsLq>+O;nRgSfiQ3!*g%XI9~@~ z|Agy;#m7R@kK@W7xYUZ)6G*->Iqsa7Z=Y=;tEnIe=9@8%Q0}pf4 z1J~Kx^h_Ep)6<}u5IFFrM}ekfYRA<;t+X(j0oP!Hn5t=fE>o2neKuEx^TjFht6XvL zg{}7^N^`FZOE%0As&e1}>kZ&QO%RoJGm)++LugL3o zn47on^*(P$o+0B%WmC|}0qzhe088x=T`CJ!-u}l=FK>_+cI|&&c!T^6mxm1URvzZ|fAkCX ze^Bjz)kI+e9}g16_1OQ6O!hyYs6pzTe}*c-uoB&OK1xs zjK8~5;n5f}d-un%yDR;VH!3#(&WXr6t7Il66hxMuD0V_l`F89&LIAmWK_(kB8yVU>UO?nHN%D(t2PYBDPp`XwC;vKIdl@LDPxz8VdG@V zXH$#lX-`I!6rW6HDS>K)sqPk>_QVYQ4bHi6!z!68h4J(ugt;EBq=sbgDu|yJsmmk| z%P7-o?LX+9RrblR0h%Nr|F!!vHv85_>*O{bbleVX$O}QJ>*YnsliYe<(Bj@RNi~ih zW*v7SHAU|C#2d?=2A_IT)N@27uTji4T4UDoS{XgG{~4P0KfxpRwku+9I+ZJK3f|_z ze`8w-Z4lNRK$ebsQRuks9nae(gCEfgS3gzlpzHye-Yk3QtK$S1zQSb|@#me7z&}tV zAn+v9!WbX^BKYS;7XG>H4gv!IJj}uWv@d}FZz}j#O%#%~r5E_`Jap;5$@d<_pQMTc z@w2-_e7jp4kwKy{WRQ9L8!~n<<5R0vx>i=Lh+BmVa{gb2eInLahUWk(yz!+_(Bs8J zxR~iBBjE#m{I{;dVCa~T$1C^88WM_q3vecNiC>~=FB8KI#GtLlfsG2rN{y@mLSaD% z$tUhf@!vwaryiV zhB)>4!9=+@)Yfm2HbDo9(1LY^O9r}+qyEFB{^xc5H|zTEl={!3e%UxaNo7Ygse>G$ip-%sg&w@AP5{v%V8E>jDy0GvL#Lq9qcUP`FgE0Oh%%97uW&Gm}m>liBP=Ny*s*WE2ErjUf^GcGRX{U7L~ zg(0^#QuPU@b#*~{0o%(y`F`LN)bTU--qjj)+{%NF24w6SHm0;j?WIZT%%;~I5Jk{Z zDh_KML*x>7m4lLdQ%E7xA8s@cG|Ec?CQ7MntHuCVHCiN%mYgPy3R)T|jh39iZ5~>B zVz3ObJ6c+Q3eU(qWLT((ZFs0%5zoCZ>QO^^yw z2`wqv!`!@Rm(>5xLq?lL(yys?)+#N@_T1Y;vQK{|Z=BMZ7^OV5O)vG=#lXWUoyCv) z$46wTFXN+b@(pdSI*MIjJuf=|t**{RYNpQ8Fh5u-^kDvOuBq_X{J7tz!W_&GRWR>k zECTaFbs{(Vtq1dYCm(jjcF8JUcrgEaF1y1#C*SQn?1Fg=rpuJ&q86-BLd2 zI!1aTjWcTYWzQ17_y_|x1@6LSlDM;OZ@kQMNj~2HNpKYLOn%Ayxq!O(?O*T@&zbx4 z&zUE8j2T0-?t_58{Fp8t-Xhx2-VF4y$@N`Bm1eoxPZ0_Z`{WaWR)#;naS_)+KB zr6VxvLWW+UX{#8QSbc=CB_T{9b|6Ms%z|>P^~wF5Zug!xKL3-t%61kcW(}*15qrrO z!hU5KDV1DVyqGm*f5@8hSzaC7KI&GuV)sLGBTNQlVvyD4K^>9~idp zf0=3{!MEmKZP>T9acu@)oI6FT$m+r2snP!ImyBN|0t{eags?2Z*|mt!lb6E`+9k(l z8yKb<7_1ry1viHL<2I%*7W%;{UBV6!j$sp^)bR@*JPcrw9XEnztM%V_dWQR{J;?5K0G!G}@jJfO0CTonPiod-oFV^~K1F#zOfUpn{8U>@qF%_m^I(WeWiQg zO|YtDl5&_n|;bH}ak(Dy0Iw41FOZ_3C_lzvQHRJ`(&q;ve^U`XLSiZ%5*8gM*!W z*TqUQIOmJfMtb(})oAdX(N9sEM;-h#*5(ye>?3D%T~{e@|6+0gDU7vw)E+&Sw|ENq z>(T-V!5-{mX1EtWXO6*ZCbYUg(^bb$eT@P;kz{SLmJw-aD0^P)Z&17 zl9^iFCeTz6AMCYMC>3NHL)Lh&y_UotRgQ>asTSJXxu~hDt@EqG!F3_~hk=kk+E(AV z*dJ;cACR7_)3c*EM0#Wnpovbg1!``YZKA+*M;#5^NrGvSsE0=oRw@%8*`;uWktiT$ zRZG)n;D!Ch@qQ2%#mE}f6tW>=% zB~|j8e^7$T&PRGjpswa+yVX^ z&be(yEEsK_;jg=m56sMjvKSsdrrpo9`tvttf7cf!{e@}i5Hq&-qpFKi^F$}yW|&rC z)Sm8(@nO`S!f$_W%ghWxTMUYGNqjb95(7`1_l+}@;=pvQGGI>mw}uS7Ybk2mb4GCrBHNzR2$M$|ew zW>2K)3BH(p4xLi3O>3}89$a)spjw8+rRbsh0q05Jt!YM14V10VUfxyhxlINEXRHZ! zxM{|OAYaFimgyorz0a`w;3Pq2Susd6ttmeAG-AWGF4tu-^qXK&vSW+tFYpq>vCnFY z1#5lrQ=-;pu~?SU!L}Gv2*MJxGcDNQD3!U9Y@2K9vSjKw616!p+P?m*?*M7F1X(xZ4Oz=Ql$vslf~Md z_`PTuVT$1lRKG^iubh2PAzFq%vU#Facc7EDBv!wWqp0VQ8fkqh^uM44@f)s3X}TGDE{bSuk(_qu>6I3E#Q2>A&_?+WNGi5^kKIUMupBlwn!2+A-sRt^0%(HUZaaEOH^ z2X%C(A?MFz<&H?qgK+fxGj>}IHc%KuAvCVsyD3(tn_U(F0bqtIIZ%XsV9e-4t({xr! zbFw)F;_r=$!Id1J|MJ->5-mhDIKJy`-@Rsk-B=FYj$bLnpgzh1hIjr*ZpM`|PI1u? z7KzNhl6N>k6kVLE7oqPOJ@)6w6=E|KYlGodF zc;CDAdGoP{^tQ`-^{btC3Mu)fzpf8G2n8>!gqA!2=!P`u2a+{_rhj=YQS5k)%mi*|3pHJ-8MYE*kjz!Ppw@DaV znAlEh0(122|IBd{3VypY`Tn|z^b2T-@LBJ$y9!mT2SXZq{B=i38Bw`l)!tmvkD^E< z|8|GRjlX-JaN}=rE8KWz+;8743lIvv>R&oWN@f}Io&Ke8qTENy1R*Q3(t0mq?YF1R za8Ep*H`aOuLF8dpP$U7G4g04;lNipKH+z>+=iGx(JV(TY53$BV@l1dS848soB$#Ol zvBafEf(^Nuga5@DGR6Pm{30m;4S*3ZnnGy-w!0FbavFbSjEERq_llpxhCU}BI;9h^ zK(qD;b2^2m`9cP3w~{dsg#bL3lvt&ZohU@OE9{;|8GPNkMQynk{|>hQWq9kv%S+?>>*-5oQf zSE<8;LjE9bFqG6`5>khfwxdMqaJTGdJV=p0v~y)xhefR`eAdM|sROB25F-p0Nkh!K zl*T4H-W?Rw~4^v z&-_K;a1_!5EpT9827>GTsgiC_SkfnIMczOFYIo6{Ec9D2j`p#4Q(R5?9hf1=0y;eC zT0jAtuER}98_L{MCLwKDvOx|!5jbRzKHg>@3UvmSIYV~9$)3J4c`XQLP#yy-R2{Bm z-I-YZknlT7U|og&Br2=3({ymZfRKCr91#7(eV&>^CfDe5WI%JhcZ^IPiT*+15&JQXPKG3r-0p%981gvh%B~qkaHCe*nHgNv zl?Oii+3G_i0Wekq3^NBdaakkP8^L<%27bB1S*b^QIDc0hAgVM?9$s>B`nSgcUSDbo z6r+W91T$UX4OIXSi~gs>5Gh$QDEXA0fd0)()kVf+{F-I_`z+o!gvAp94=kQ(q1+oR zr!$*`2TSzWl~bY`J0WA_(cfYI9`We0Tz)4-28W5{15NIXZv1KPtWt!qq&N`kOGWp4 zq|3OFMewGbq=JgzP1~zrvw1d!@7&soNCq8_G0DH!_<;mU@^t(}0q| zk)dSL&DmpTI|nNU#&9;AIt>rVp~KG4obOYrpsczdA9&V{!DUZkO-n20MR~}tPp#4O zR41xh{073-huN-J>Gah=n|??Nt@I@d_QLE&Z9l2Ot}qiCsoBztAG z+ZX#u$i_FGZDvPtSll}7v51piiaG<6c(jHhJ2$P4#@qHkp$3FAosiOKEqAHVi|+ zYQT*62+{|^t;nQ{CvhZYXRdCbfG?QYhpHYmb^X)1nJf_E7J3<;lgSJLIU_?#xG56~ zNO8}U4$4ulS`yGvB6_fZ5Z^eCz9!Q;-U>hK(XfL{Fv657_00auM3zizmWa(tNm0~@ zh6Vzw8t;`aL%98qR^Tm$C)$4mJ0lhzeNmo#=Nv(B=gEhG2oWxX$Hamfp6O+FrQt|O zhmYMSq8yd>4RUIQoM8x^@0LqAxfOHys=hJgkG))YQI%=m*1&V-`Q|Z&7jB&uN1$ehF9~MPiY6f8uQ34l?NiEzus~~YX$JQuP58nK#+4Ndb)Hyp7 zBa=o2tk{XO*lPa;jF_3tW~ym7&lkCRrHRFZ$}mzLMrzBl3}9^%hv`>gtNs>hhhnYe z&@-$EI<#pbv=x>nY+dFFZHpO5`b|Fc-{i;`=mY}3#E-mQT|%a^n@h~5c6v0SFc#c= zz4WL`dL##`nIZI7V-?RAf0-V2AmK^7OGI~g*`mK0)-(`K+ML?k%wojxJf>Kb{Un~(y$o<`WkJ*kVW_z6e5$GUl z#YUyC0WDC$IC?=kp+zbBL8k;)lK&($;wANgX{-3ZA_n%?33?~zfdHHLJZ9f5cZhwH zBI};yT(%P#@Q^!^0q>k)2AlnLf|kt;98JtV4|3=*)9MqmzZSDEhDNxs(&xn&7L44z zO4xp?Ja?lJvyN0w;MO>b3;$NlkBF#I?LlL4N#B%+`cAyjH)ciWEAm9=Y6p!uF}!hB zMDM0>p)!)9aUU&9*ZirEUas8!{2kpuU$)*slL zUPw8$WJh&J-PuLy1%KyW<>3Pmfni$P)#k9HO|yafCW9a%qoTn!?IIx3XyB9hv?03| zq+wJFBPgI03O0o;!Z=U&`J?-5_9DSXdOb^7(@7|{%MdfP);zITk1v(K~jswS19bz2=lA^>=1UJ z36(x0;jSjwYG$Hy?#-X#8Mnq zp;3$4fOG@oo~=cFS3P8G@Kt6_$oQzS#odNwMedk6=S4T!pO)_5R4_-g?SjnEAH zX9{uJ%jAb>;ri|u@-vxJ%LC~bvmpKA4D=^=8t8)S{y-%PtfTh%P`ZQ{gYN%&)V@T6 zwPC4Bx>xCebTgs|BF;iFTjLc-8`dZfq)qF(9w5ysHfxYJoSSkWZ7&l94@5sVD{S8+ z*W(JK&9hQ#J-nafiD8dJctxt7=x)Qhn9K93u@wjTrWQ7X@25*DB@vN0#h>#vOR&Sg z;(0Do3#)lOL|&&@&ft#g_em-88x*0BueL89mRdZrAl?t2vqZ^@tP86_F5@(s=Z83+ zrzp<#%MeYgWHSnxq!R<>2&I2XkLpQzRcfK^_|j)7cod~mFB|;D8R=8Ni(UE<#ec9V z#U)h1Q`91ezzbc1_hTXwvZhYu!%`5j4+Nk8;HDH0LaEi2-kO}wZOkiR20LT+jk>HP z=ebK=n*K>`AWsVY<47X9+Xz;D76$?aeiK2ef77&-YATd|~kGB{RkPeI|@xR6(GSGqy3yDG)*lSFbC`hr!Kvk-fnA{$o3Ih6ig9Z4 ze4#5LWhde|@a3R03*vbHYM#mjCbtNG{u9lgtHs$kBYg}I`Q@5W8^Ehy)>lqBmIGYS zDS)AFzZ6fCU-AEX*YeIk|Km%9?}S9w_&n?XjKTV*1EPjF%s3!^RwCiH-h0Z&mHQ@^ z+d9pNa(&hXJ$^E;ywywjGAaM7zf;~5U(J9lrwd08AkMKE4~S%ET7}Uv##MNVEE|&Z zBXzy60Ebhj{3|pC)riS;Q?{?^rvlQ}DAkq=W}6|Bsr6a5KkJS!8F>LP2@iKZ;=_WF zl4yj814qLPvO1)z3#8u0))(D4{>lH6q+`O=O>V53w)vG zvBQ`jxIQBTdnafpW)-RIkWp)B6#At)X6=XuwxWF!Y5E?2XObOOnvO(IP@pPpytRxAaM}>``lK6&4}2WX%j22w@UvG?%Od z2n_q?THYHL*39>ElVnoATvT-ed)^hQ!t^p~e4tbQ51ppwEFOV-Kh zPaq0Fa_TH)PqAb=Pdt6PVkN5tS_!6&PB*p2;AKHPBwUxmIHt2az$1mYxvt0(5ptQFab zRTo{H{##mJ%cYj1WbqG2e^qwbGPN%21--gdiN;u(8qZsAi(#_|$aeTsSzojfVzMQ| zBs=MSns`OMN7a|B%@8agMkq7@o1fNr<%JN}>d-l7?iaO2*&k6M+pP0^#`Zv{?nfD!|1^-K7DH|2cARSDc zB;Y%+EAvQ#&7bRCgj%Fy?#F+nV&+WEWz>2Df$fCU>NnKTvH1x8(ntFX2O4%tU?UoA zUp!bvZ*MnjTMctM4mrdIMeJF$aHB8=fi!Nc5OHD2KeOa7Ns5U`jGg%(v#1Nmtd#;m!W5>s6Qs-xwv5Nm1c$yWK+dPF+M&%^rX9{saQ z|2&|78ud>*KU9?rHC(q}tBwYTtn1JdL}nP)O6h-#3O_7!yb97#@~vVW6`O%Zz7eT_ zwL~^o*s^5h!j`h$X;~sTPL+^9h`eTmB)qq#LgiMjW!gYBjeH^Dfktr~vwl_C$-O{= zR9oP`;ePd1$+;(qWkF-SI%2&adQ$pfHofj6l)w?Ef;H`uZ}2nGxp9c>Z0vn#0%M@# zC?v)}WMAERU=Lw#NfO-GbOsJFiwKh_;jbIxN9)?CY11jHrasi9YY1ag4FZi=j~0tZ zOvL(Y2$)dA4Z8lNR{%}s614@$1v|Izs#<#0aSYS*lSK0fxjeZ8W_`2>m8sOGJtmxNF;$`LtMbNJ6@^OzVxOlDsfyBFJ}5>oD?MvffxD zmxhmJUWpIF52aq3TG+sp3_MZK;|XdB)kKnc zNr8)9-g2^~cAPNL4W zSa_GN9wz5PUw45_oqZI>OtQu><39I2?1Cj~=OJYpyfWcK%dF95s`AVHNZL79l{w5U zQ)P}6`>AX@7G(reDvw%`lwt}p8$8P1%!_wxkks?}KNI1|Xz9z4We%aCE-%xD)D%Gq z3-T1CA+$1yTRkplNz zHMnVe?D@8V~>l)P$-EGi=tVJMa9!>|LPHL~v`{;}_Z>p%p5!ITNKm#76e zkVqPeRs~VRz+J}1?Q4`I1)gRiSAg)r8A1ZrwnTj3-i$l;O;pPg5CrQt6HWCWL3L)+ zn|%%IQ?ux0e3fL!*KD8QJoNiTqR!F%v-{YT5D)Ke7P)+@pVF zQn?d^%TJ^JY1cn4KjDU-%JPYwY1CRFs5@%iuv`Dg(G#|AYgc#DxdMBd5gzeLOKK&L zxaI9{1s7RA)tL8bDbbS3V&1j#38aXa3>aSKTI*>aPmzo^@-Awn1mzmmot@mJo|d}6 zx8mIywaeeE6x*on@sq>W+a88&dRy@AFF_#S+YN$8oEw8O!=cP}O8kKmA!lJ_q5p~J z54>c|+@!Eu5s{a%L7=9ia)@?@uHp`UD2j&(kb(ciuZ|Eb8?rvLe=2x;#2Cs(s;8nI}7OzgBSb_V4y{xT8IL^xr z2_*!)yoGH}Z03tfWio^=T zy6j+uB+&5?>*zlY7sQ&09}U}=8|nNiHCb!<$?RHLv%TS`Qe(qUH4+6Hl@ShfHy%y) z<<)G6064W4Bb+ie#*B;B*X-@s)wLxAmrlvzQo}8yPdP(~{I9^Y1@1a5VtRj@wVe>Q z7I`EX24wYI&EC{A3I^uSv|dHVz+&I{tqN8|Huq8PJd~wE1J&tg{Fd=9v)=#2rw3k@ z#NSPC_ciQ+-;%G7M&^lBrissszvZ4mN$*YK&%wK;jk!OsAtihxp}_>O-6z_G99~f? z*L!d5y3VwVv&l27N3wwq>@{ZYHQ;cJqCdI)B~g(5%)@-D zjY6=tO>)E9-gKnE_V`YB!X9SMKEIQQ;+;9P)bx@(d}dWI1<%VC9MoIE237F!Mpf`5 z%urzblibkav8v#>PItHZ<~&-Q)Yal-d8kt$lLjeW#&BgBRo8V0?yF&%cY@}?U0Xn1 z)q=V>#SfRgmW+J};kRg3)Oq@LY0XN^0K%t$=?q~jU-&hSF7(c zOj$w4Nyf~N^0M4Gu`On5c&I7Q|lY2HX-jQdV| zQfi4@`R7Nj{np))qDzY-*70HcPC;U!roWbjGFu|UKZsa=n`noRs!)ivC9*(@HrFTR zXM(dZP?dG`Ap9=1O|};vRYCU`OtcrCSuxo@_soFl6n|Ssr$FP#C-SCDyosT#oP}p) zCJyJzi57p%B8+-JHn*jt!&{{&+c{P+F?HtV0-W$D2{pqh{dQr@9=MKLKbAQuii)#$ zWKLMNEZ-EEJCHMMGMv@xWkx4v!e<8d8~yOtBddjcFLo1}%!jFq3z<)b_$-G%c7szuS<^vTGeB9h4ClU;^hz|_YQFq+JCUz5XX&l{ z>NH+=y>c)!*^aV298D9g;%#cJ(s-3+fRr=KV$Os-b<{f7$+d-z!A04qN8r(GRB}~yCCL&Iz{1gH&2-Ki4 zHnzDy1~3!}xPCgrMFp9e=yE`HWwL#t^kQI>=?qzijI0W3f>c~vh(+$NyPx3^2C`;* zbVQ^|i@wn7_QkIjiJRHv7RTxv=Z_W=jp0?r$OG1Ql09-z%o>5wW@o(rBzyKC-y~~x zKcA7oWWR}qE+|8yH1`Feg-|07W|3nsu4jW!Kkj2RrAHe@O$LntQGJc3&LZhX>P3{% zxa(}gsb2Y8M4-lpwIMOzHre5SfqoeeVz4s*vO9p3nDtW3DPD(0++TMWW58jcW|Iah z4Dp2?Wu&Brey12&*Zi?EfS83ufVheK;=W1tz%pdY68>3@AzH>5q7{sxDdsShsYYIJ zC6@AB7%w5|!u`}R@UO@X!yK`Bk`pfMYcd7~Ja1^siLNJsFj?jiSWdQwY@Tcl*}}XF z@RTv)KzycYtyjmpxvc|nh@2SPnNm67H2#n^}jw zKhwHn?oLu73@(g1Q_-ij%^Pew;Z17Jr06zXbWs4JXPg{!P4D)BveT=|*+kI0Z$#~j zDq_~2Xi;0-H=(7tLW-EFPiSpIOSnR{reyAoWmsL$-H8)cKhiZ$ptV@QpW4sNGnxue zT^r5fvk2i^Ks|ya3)0*4>oTy>I?q=+{T$Vtw?t+G1eRpc$s8Q$W= zid<&EKC^711@coX19B|$feh8k0GDO5Bnq;?mHi4yTt?!P^D`z;6&tac$tdgsscY#B zPd=6D@z?#k+}Xtt#>G1nI580-5(uDw*dvnymgaLd2vl0#ee@8Wu(+3xn5GTBSb)AY z$XePAHU(AA=PUv;fNkJP?Te^g>__UV_eInW`+=$OsXnd3Q2PQEhVDHs9Mbzw=zWOZ zzcX5z!Sa1D-OsQiTIPKXbX0*+$ny%K=;UKtjX}#QG+}+hMIdFB9T@+q?Oz zT>Y=Eu<2Y!wC+V^^9HjCcW8hzTZ%tr8aSZ9Z9)T;xUcO#2u_ppN)lZcEn=m@XUh7V zStfHDQ1~u=`o|d%B0DyHJE%@yorSKxix5!eISbqqhJ&n@Y!Ko=IVWi`GX6JNkDvf1 ztcchn_A)t5LiSprwD2z4o|G= z>>HAN8ynGjr7P!plUo)ldQe(254;z$9+a12B3pzoVxN@7-^~>n37lwOVFirT8JxEQ z=EOy9OvK1*h%;Zr_X3{%S%8=eB$oh!O@pe;U>g6j8OO1H33u-j1S?`=SI_8 zz8n<{eFK5hxBe7M**q9;zwYk0;_{fl{5yfK&0!9C}b~_fi<}U zL&0T9u6{hP=f?<0+{_eYQH`8faYQS5b>WUaipn}xp~T3_MloGkMMVYvn;X;T1E*i` zsvocLHd2YwP5Zp*qB%L7QFtS`eh(vP88=;OGCgi1Zm4s}bZ!dBv>Uh=b6p`bW% zzKlA*lplra;#7z1I^(a0o1&H1BqXwtCB&@3ntoCr$YB)LB6Kp%t ztTK%5d;<7~sX>%&abz5q@c%>dRV?_*+~cD5ovH`H-TwNkD4-y{MK(w*xNa^XI`&0s z_Hr5W-?Bb}4Oaa*su?O&N1j?Dhn>uQ3`_9jE$6;bSF=uFj`y1b^$J!s{1^OIDN2PU>lyca-3NCuAdiQ^Yj&$O30I1+H3kw$zkt+A2y$ zod^V`CXjc$oXMbJ`et@)1vn5bSzjep*g+%s`J!WlbCTq(9*K)PJqL`uTKJY)0a;5e z^H=?pu5`5~lcvYV1WCsvT8=ZiBSnq99(-XCB}1wHM(VrV8I+|5m{hm~~#LYr;^_>a`UmiC|j4 zEPG?_Sud=^j-9ObK31zkAh96+bq}C&iV?gflwt(i>g*by^$J*(6C(9Fijnj?FbPqm zVnseFwpir&RC!jA5Ge~Tf|H0I75*b7dyVZI=Kz~t?XT;{ZSCoq=92LDA&vFQDHwQC zc2+H64GpzE_@4VykI=pReH4_s?A1ecotgMyM(~pA_;J+dok93sQAK>6>I~a6J)iTL zxP#YE2ptdumjbq0O{QZIchHdqp|$)eAox>DWWc+jHFCqpYiIB!4>wB44I+rE)43pi zN7TN4mfG4e`>bei?R37HUswA?VaHPSl&iGFdsS`IYuS=QO4i4D$bEo&y2LRr7$f~M zLLdL*3nqB|DyCb0-7D3pTIp2HKFU8XK~Lz<&o$!N$hrP2HnxdY7$ zRcPNV{Za7dq4}k%gUltnSS8Qa^p8R?N`Fz1AU834T-1;`&-0bqxd%BsjfXX5ge98P zg$a#6CE@&PHi5NGXN*urtxFqv`j>Ql41&xKmeCQ z`>5cyd1Y3cqKekV50m@3`^6xGX_fq2o&-YQ;X%@Ichn-3z25ryWWbdImA$_@hF6{% zL;Hs#=wt!f!uBtOG=={6-|%ZXq!_wJ)2R)Z{xI8%a06TBmB34OoD#B=Ng4U}y!mdB z`8E%H0K*uv8ss&fN)f2gUMBkino^TJu8Rtd1v}=xMn)!#6lHGUKj$B>=XnOV^oMgU zab*7Y>AA1wQH}okS7{aqQZje@XBv<;`%_QzSO7_lCI?6_2_Wr}ApuA{c5eh-+Sj(Z0tA+UP~CLziJAEA=-ph<4Ad-EVk5iOmHUP)VmL~?J$N_rI?RHWuv zykd>hmnm4w7Vh@Rh%bJHbO)mSFI6hEfIVS%l)w$*Ow@uU`{aID#cy4V|4Y>vx$2AB8t_f&C)(+yxqLEdH5MsorH49^-!;pWJVfjsl?NIZ$Dwt4} zCorIue6^mEuQrc=%Wwa365M9B{4pO%CISWVA&AlgBJISehd>YVEWs&=duRQd^N%+( z&+^_k!7gr>jtoRV=3Yg|V6@S>yqS1a-sav+@!l|e3;DdoE4!cjZJw@#MMY#K9gwL9 zQ%It9vdC1(T}Ol0OE^&~b}QbHd6kUoUC-4mYP@W|8!rY}*UJbb~v|1JB8Y;4XsDV_8h{Zp4-4BGcO!NyK+!#tidMGQcfFdWs zEu z+{y|e!*+Sa$xXk_S!)mc4$etX_U19}P)Jl7x)CNk=kkuY| zssm5$V3)Rk4|dyAK4i-K$N*5BiahHzgj13^$FPr4>IlOwnKuB{geV<6wS++5Kh!;} zV3_D0(uk;f=pNof_n@+Xitb^c(mjOjKgd+*VQT$DO=Gy|zhwTx)_=)6xHOTmX+^~5 z$nf{+{sJCg!|^vy{~#&}WWD~1iO#~|RURp>)DWV9Ko!B8|0~rH?`)F|_D`xIMz?hF z5?@{ov1`-+tQuk&^)UotntRc+^iu=;SJV&^bXU|65~oyS@A8fRd({vp5@IA%{_8zG z)DXx~BR;obgHl5Tv>L*vWPpEH4Iu|Os2(16M9fn|piIEGqcy98Fxb41qE}-vg5AwU zS``tqM;@J1Ma1mcNBbsQv-^uGLgeK+RfJ?4P^t(y%tQ@gDs@Efs)&#NDOH46Jl#4QrYYbsQQxceFHxw!t^S3Hy{nCB}XcebXs)kApiB zKr_*q)lA5@vA_iT%r+D^1d7vN;NjHC^9txQH2#~$TVx1vZy0Y~@HKV3K&{nJFzKUK^dOov|8+6fj~T&D816Jr~@v=afe6KK(U z(M}wLcEWWD;vu@&BXJPc3JQtruCI>kO;z#Z{&}j3p7v}@{xU~^QQ5m%Yh+2H)?-2o zMlGFgRr2NDC$a#;TGL6V4ePNob@%fs_2W!Yz+CbO)}40gSOUHoH5(19o;cu;DfRm) zLiaM6^64*#*-x|N1gN`kfXv*Vp;YRnc+FnZo*yu5oOl;kBRCOB>ytvvNT9g)A)Y)I z&;4wmJZwfhmr0zU-*y)xxO=+3h>yVz>Z(54atc0NnU7{!F-)CueE6iouzge*PP`(l zS@95#g~a+js^cA_h0H|EHx2shzoxp>4ttBr*(o_|80JJy=>mR8h{HI^>GG@`Da{hT zwtjOlea%@6iv@9H4*xHxZFtP#TYgsPpPTq$rbH$n|12FiM9&Og-N-Y;afH~mPD2<} zVJv}+7qYblLU+x>Ja4Jr&EIeFp?kC=VPkY*uN0K6&t1Am*gEY3!-Ph4h3;>IdP1>p zWnYziV|P(O^SBR@TOKCCZl4P1wj1Tg`Vo9=y>ugL&AG>D8RxQcJg!~o-uHb7atM1{ z0fQKI)fn~~q4}t@j5bPDKNYufq3vp_P7KE^Kr9CbZf%*g?~H_I?Y zI3sHdb1#5_@^LVB<(s3!JEPY16|tg^W6A&~Hhi-aaZ5rr(FdqbrVgep1-KDUS$NI-UDdx^BO zMz*uwvxa>ZJ2bm<$Mj>m30aW(jE#iii4{_cQG`&6sg%e?kMlTce+$yJAJ1-qKS@3ZG686p?G>iC?s6sgZrjK1Y^51vh}4 z*yWA6Y;2^R8}duU&}I);(h*UlUY#q6OscEo+jHB!8I#e)$AWeK_BoNY(Dx8-9;>Y` z0Zlq&lIuh>^j^pA=AYW=4eRwrZ}Z8j{5_O#v3zMe>SV6LLa6lf6N*p- zJ|pPP*2lT64|iJ~^-)_L{83v`0#RGj1-7CRur8~moU$&Hj$)+(2nn>zPABQtF{ zn4Vd+Gg#Z*+(Le zHFZ2erP$@`Ml6W4=Wrg$r%?;>S#1@KJiwj*!LpZT6?<){9Ck6VFF4}8m|es{%>ox# z)2rOFaw5v$v`g`HrfenRc%Sfe>w~mJ;lBu_Qm-#g;)P(K1VMwc*g}+w9@btNkk$XW91_IC zjZaRFzRzI8V;XatArQ33a2`{8%z7_V{>Lq(8{znX+LLWSUKjPC>qO|nMD7oc1Gn6O z2Vg5-G6!w^G>`ckfnilpYr{g$YpKZD9zo*P&yYYuYlQGAh>u9F5qd}x^7cezA^`Fp zX3FR2n=a@*E5HobjX{dwUiNFct7QqB9zBS#lE9nr0AkT7hi}?}RyLhbA2Bz?wrkmB z?m3*4pws=?;d1PBo$Tq;^Y(M6mbulc}?ZfgQ>v2#nkR?VO-mC~( zo>HY@>~Fapw z?z~Bm@W%9$c$d4=@zHYEt0&FrJO8F_m*xl@s?>yztdg6OGiAvJd8{3O@;(p}&wc%3 zdeQp_u{myCp<*d|DQ-I&Ue1=)>b<z>FM&L1t!&YI8a+5baQ;DgzI+qHqt?-#4l%Cwjm+w*Juj&W0uTa1L9r@#pzyl zX(p2#orviDaz%tP3{>VxsC?40X4$B#sqNr~vtEHSGUqq5kDb>@NnW)TL4ZBlZkXBeh36dRD~C=B5OvAf-2V*2RR6(*&F zF!IX2tbzK`xT8t;*Wbx?zGk#(?q1PI2cp(C=s5Z<>QMac&;x9P`i4utv7c0id-uHI zG*2s1X&jt`*VwhG^7 zGE#bdVV-1A@}xUIe4eOK80}W*bHHZW^r)@eE%6CzO)i>IK-P9%02-Wl%cq@gIELs- zTN|cAge5?$A7RRed|^@`=8q7CeiqKd%1Kzg#R4m8Pf|*TBh4jE?P}T@F`somd{i(H zud-HHQANu4qMnc1w@qQXVs@8$9{0~4R#--t6_(T*wZ{cM8ixn;iD+;{Swn1%xx^gb ztQiJRk>lR;N73@=&moO4%B}Os(#Ss|j8^9zC6S|L14MM(c7Is}s&9(u>o@xc75d9! z75+hHnZImWm4DFGfWPdbQT{>St@f8)ZukdXTFX>SLAJtp4T1)Cu`;;0Xl_5~cpNp&^h& zQ;NG$BV|zuoF^7G0ZshKY*ylUMK=C@r16(lz(4tyA+NH%QEE`^Ngm^L}vZr1+x{%5^MR= zqB*$5EKq9Q->;-2NS0)v9>oW(agVzDxsV}t>4CG5CACB(DIemaq3fue&`ijN>VA9} zg(S?sjdjpiERHrR7AfmJ@2`6gL@h5-?@Q~Dr}ew2)3h;v+yg$5r&%isdHIxT_s@EG zd%!(0`>>iuYg_>HMfdIgpm-D@(qlt;_0Ow@(B3V#toLxypZ+6n{%mQy_ytkei$De1 zF4bC~FpIx#HhtFEMY18(E>D1776^LzxVGWPBxjAlQ8pqa4aC@N;Fry$1*u54bI{3o z22P^enewHC5P@)0R4GT%18UUcJf6)ebU&G*NCojEIF{pYM>BKmJoYf3p(YfLO>8z3 zKH=8(*`@LC2)A~3qi}02D$h@VgA2jML87r`b&URZg-b)nXgZHp%F@ZKlt4UU>Vj0l zej>>;k56mP@@dK@HexN4cM%2s}rqY{kTdkAY9xtD(-Yk#wnG$?j0YM8 z=aPR?D}IlGkH&K*SJ^k@j^smRc%{FSZtAb)NG94wwHlAAWpe(J{D;Bvsn8~ZJvfug z3};Rt>W@4Z@(&96IF$ojm=zq$)m%)i;#{uf;=8c0i>8>?wFVf*7n5u|(>R|mhr~cT zd6#l%mjl`jU5A8dZjNpFK1n!L-YiY0L}S_GFrsY4tBl z%Iam=E}6#sjhgg{}J!G3A z^bmZH$E+i#?@}q{L{q2pzlPx7)5#?LmadE z<5g1u8y>C}_cY}M<(v(RvT5!MN*S!(hz#XM6f1fsHlo8!ZNw|>bG*{F`cs#Jeq&C! z8EqZmj!b*V7SqB(YY3@@#*ta$ZU3_Q%rDYk>8yzexo$bVq*Himy@iyW4sTZTg_}|_ z2O9o3;u7#p3Fm=*ShR$)hnuN`bX4x(!K3B)JT=Cg#pHF^h&J`0_yM%x@>HN4w#0W$ z9JcUvoa}^$D*rj$_~wy(FuYmyQmnt)CRoLAl4ouB#}hOg2J`pPQ#RD0*4udMQ^Tdl7oUgKQtooA?4Uht>6EZZ@3@fCd+6SOJX|2`) z3@-^VgpUCj44mGc7p?NSjWo4g6>XKG@M@mR+h6aMZGDZ;C|q17EJ0V$^@|A*&Qzt_ zz~}1XA#gADf<%{ACfWg<*TT!st(>L8!|i15NShq9CRg$Q^hm?`Ga_}&tWsQx2tPre z!^@{lAXHX>x2KCwV$8ZEz_XJ=$<2Mj4bx6C?Nh~dfg^cipabG*E7i|PzFxRIc2e@S zgZ(!*#7>IYN3lJzKo`tw5`u-}YpYPX5=zGYFL@fn0I)S?{eV;ZW__QmW&NYpnASA0 z^3cal3(!o=AKn)7m-XR*8}H*{0)pg0lPkC)DW`wX`BhwfPtL(hMsf9hIS7AXpj>n= zqHnktN0y(XIv!~_hmQ9Z=%C|`s^hIPF*8DRJ-mE6U%hcK(tl65VLIJEPKjd9(bPHe z>6kUOLI!0o*L^kV`egjdyskG$*U5R%`ac+V4Phh!J1c=ttlhhZTEn0)KzOx_h)a8m z;_L~|wk34nrP$0(q6m%RT^%cGMNz2&uU$*vwUW>u3gHA!-_CMNbUFMeL(L%-dyRl= z4Xg)W6MJoCnOcKsxXlq7M#o+oG*>{&i@kP2SO;H=1>1WDUpw)a`nz6(uMyKTFZi0f z2yf{qj^i-p0?QY!Uq0lkC#ha+4FPt9wfJgz0V5`G$rH?UvyBxU6#`DPa>v0@xFSbd?j=^4bZU zn}G%#$nB}EIEJ$uYRfP{72pw4b!scv7l{fIZ&*ZYiGUg#5R2ozrgM($q3wo0+!k4W zwt#eDxZ!MsRb;xd3qzvz!0}%wUU7rFyDTd^C1KVdB`H%(-EmBkq)j z_3xw5huRaX-C3e2XMAWhGUdlmp3w=iyug!(==H)l7dwSr$S)wA#4J64XhOvr`)Zzef z2l3d%)EpY(ynw&X;P1ub7#wCZ-(|?1AsFWbuCP!8?T}7dgSc1Pqy> zj)8ve+umc-_DOK1DU1VykZn9wt8XYbDLgq;596iTVYp8_5*P}3rZ{>fD!ec_#KgjB zRJK_``l@a#EnI>v9GzO}E(lP!IxkNb{4<|Sfz-xGjieMWw3@>0o4L%d#ZhE8?qA%d!W! zRI<9W@bMA34B9`;qMnxh@7{T~G=iFLPo6ldPMdLQm60^_*6nmKCHyDum7~p2Of&ORFGbSTOK2pnW996@l|cDpOo2!t5nZP5b)8s zio!%+C4uNDvd_cP|1d@zwv#eruJJ?ZK0cO9S&xErEikVOJjoRcVo$1cFPpwa6{FK7LJvB(OLimDn$l zwe^tlbR5GbsCEjtT^Xz8h=a-+ZzS*uD zg1nfI8`hgKf@Pjglm*j?@B132lh(#=XGY9c zVVw}ZPbf5r1Mbqt0=MX=!s=-^pkML?CesUS*>#rW9VLImU~{FIUACI`8X>9-dvdEg ziNrysy=cLzYV~BTjQeuKCV`y9Pwr@KG z<6izNHBhGi)R?UdZ)3IrZ?-~C1U_~vFViz&I=9Y}*+h?^=w)JbARaY><@WF*@dk8C zuj9gWrcIG!_D?Xxnxq0?B{>>>0hhaoQ=v#$Z56(C5h;ByM@LqZNnZ!dcp&0Y%3EGW z{FfryyK?3Oba<|Av-DF}eBuD`G)2*(m)Am-R0C5S0ax&QwGomqbu{EL8CX{@5DPtc z*>Uv3fXd#a14(4Vg0Q>smA$dDohfs#xD8^{K5vTbJ3nK*4Kpzr-z1(RT__{$hn^Ku zPL5LBeItT86$}L+;2#h$fim7whE)8d?hruGcD{p`gY*$7z^786AR|!li~D*&ftE25 zG*=~tjhvoQ6(TRo#H(`%6KdB^-cF@j=1y!f8EddX9{KAg(7_0S2g;*%>G&7KICB#y z#cB#=wuPQ3#L=MadRPzyDP6r$veqj7-CR>LywYq&o6K z>kNX%lwnYa=!2m5N*139H;e+mic+oR1_3VqV@DG;1|Bcm010eABlMV3zuYiKKuy$< zg4~SY=!0V5RNtW*xvh$qF|B;S_9GrpbFhkG0e)v=qL$RU};G}B%on_e%B={zGOW^+g^w#K(ZXLK6YrN0VwHf@oSE)tgzM~qffChVp;4fqT2PPy z3PQWHXgVke90S!HNWTlksxE%U1*;;ELB9G{_tjC|?=v^TF{T5erLjj3-X`aAhEu_6 z%IwN+4npj37=SC4aL5dPJeQztrGIVH8}1k}qm@{ud|r(s#;BysO=g=a-BRnLVR%ZV znj8HTWqQTZaku$9`xPa$A>jf9eyRZch^C;EW$Z{B#%4F?MCLRb} zP6GwSTpcH=zOm!M+Al^na-&x|=qRW=po_x_xFXGs8;et-Yp>(A>&<}S~D*7KZmp0k$W-*Nu_4A{L-8`N&{AFM3;;IqM4bKh09`GuDUg*ASyUk@YzJ(;3P z7^LLoQ4h}dj}eT+I^mwO%IQ8y>~3%Hp$F*s>TFfxPuUKk`!|$aI>e9_midzC*OIMz+kqGg0hn1Fe(u%M zr!wauVp)^N4~6rT;1LEG6)lI+ix4GZplWBST5s^*v^oKRW~o!gIo6L)a6cA_Bbcj>fhl%Q0T|BYfs;~b-%IBb zCJZMu40|SSN&V^oD)3P9AV{5v?wNRlwTpDg63PUmwvWYraAra38zhy;S@7%V0&fWh z@X2oG*4`j=c^g7#a!Lr4r z0TxZY^WO>pfTu2R@N@eKo}S58mHapdPX{~h>#^j57PY8Q?#{?mn3JA>3u%7uhHt>4 zJM;J6@MQ96gQXnw2&MvP0rSzY0UT%Ko=*fcZz5V39{RrrG(Y(H|4X2`Xz2eVps6Pp z*GBh$;REP_r?!Tg~4~Yi#VyU98B`nPRQ4)o3o*_;V zE8^s0$`x3u6^VlbAEe+ok+$x`OrDj#POWJ$x&xt-A0QW9wl?nm9n|O0%X#m2P-Dn; z7#2;!N*s%jY583$X+9g$77SPflHFT%#&QAXQvfFMC{723++@hiBvV)P=wRi3hyN2& z?&;9KOV=$ped9Fu^i32et}bb<_au7#q-z_Qe!K z_}IF%&c2X1;gq7n-DAZom!vNT`4`2j^WztJXRfmNj*t&aPisLRiLtI3hxLtlzt)vG zU!Djpy=s|A?=(aU(%KX5RCcKFz5hXc7Z~I%lWdWgh6*Z#K;(SbJ_wkyrvJy+FA)l$ z*XvY2I=d*xE+@UkG-&5Rxf#%k|Fszc2fXof_ALF#Rd|jcl9T`Y=M9tEk0yJJD9QAg zM<(h&nxS1H#8TmvkdrTVMM8Ji*Nk6Ms*r{qncexb1e=DN6eqO#Abma3x0H+P(jv^Z zw9vCz1NE*c=8eqY;dqCE1OhislMc(p!6lw-vxHu8sg4?}2^^PtXRNShw@y7Oq{+KV zLRoTJ{<(q&b9ZmO-*S>B(DFOT4P!O4(G+MRQsth6`>`5{ThS}5;aSd1J{S8QmlB(M z?U^sC@?tZ2&QgyAZU_yY<*Z8W-Aa|NKP|q>zc4F>`SkN;G@+0q=%U*}kjR)f38~N0 z(EL5HNfDqRBD087jzbbbxM&z%IEE_~CG>Fuc168~x6s{_z5XxJt5Dlm8~d9HCVXvX z5B|i%(>s`Qg@0`xLC8hv-@-3nhO;|$quvtp?8Jx)0Vq+YG0}BtRHvkbn7y$&)c#r3 z2%CZYsQ=gtF4I_biWlxBDAq8^NN` zqOnMWPwUiXxrohT0x;o|CT4Px1&~#v{uK1k&3^Gli|s1o$qV*h~K`R zAL^S_o5A&RQZ2KjLIBUpp=)k}xPRnyWxhdjN{zfMntg&XC3d&(rr38(ucnz9cGp=g zxkfqJS^WbxW3WlcVlwmDn~y$FH_M!3c=qMquDW zz==+Wj}r$7C-TA{8uk?xnQ!y@&|SZtAho*I>EzmIz$Vd%pFZ#i$v++r7R16bDK9VjXwPFa;5#xa0do@zit|;IpmVPnS z}g1>wx*8K7wMyU6I7Y{oIM$4Gj2+qI5GzEA8DS_gLtf9zV35^ z$?VrE7p39LiRUZ<%N>;POYtur)f~Axr&&_7;2kBOQn$LAumE zL7L&8sV#+3?uRi-`#f&yT^Yt;*+?kXr3}6GrGAko)2-{aHME2LzupwOX=xeCqA87g zz)pXC`ho~8Oqsl-UjiR5bC7}l_rM}>fLywm`keAgwhZt9x`|B3^iju4jzv9A&W$Mo zBVE_Q0A~xq?Jmx*KLN!%FwU9nBquhdMhw)D%qhcxdna^wZ7ET5*uVMlMu3~y^?AU!EZ91tS0-rB*}+l^N79XVA1<=}^Pc^zkg;@dKLwksigx!kb!XZ^5bK zX=$<@1iUHWLB{TWy~zomI$hJ!uP`knbYae>ha09R28AAOz?pmEg}uC&_I3flKv%$7@Iy-+I#|C`}l5E)2*CaRj+P!M^dUU+IsnHPK^S{(B01XsQ04< zTyE;bdQLQZAb&P>=7@Hv{kOan&N{DmnFNFhm!Em_*(NdZN`z2lkNqY3cWQYJm*>Fr4+ zzG%_K>-#*v3o?4qY5{GLwgQ*oiE%;VQp+kpsT zSHS@w0DGLDRR5r3gXq{t&FsAr&gVMTt49cevQ<+E50-yz=aoqaxC{OZ@Vu*Fdynjh z`eBs_uYY8A0fA^&hxnABQu6R2gx1)y13B%pFrTHVf<(L%s9qlOqNH3Z2j)W%(< z$Sf+TpE|kCjEGu#hc8e;ZBW5LC~)=}dmZ(bXe&gCHXGcVx{+imTaB4KSO|z5Utofj zVkTEc!!yK8b|O)3u0Zja{(#Q|KJh)(e(}`GG@eVU$>$ZL;Wt8U1aHvbLv5HhvO?R8SDbEvSRMap zXu(-^=!JhA>9}tp&5n7$TcVm2vZu|2=-%LA_lRAb>a&ZSYvEq_E4fS2*=mJ2McBpj z#V%4EU2U%N*TTEic(m^NZPbkT?9I#|$<66n`&E!ZmE60H=wCAGH<(DjC#{sbO%K09 z3IE}XJUp@^ILxC5PN0;aq0Cq8r*6TDKrJjr8_sP#WQ(RmK$cP3hD?I(AMDuK6+ z|J7%Gaj{{;WHG_#HF_2wxElfPot^DMtv}c2J0x-rwa@3dSfK9|j1m)x%oM*>&-Mts zxutvxy7UJd&Q6=jTkL$=Bjn$){*(F9Xbd%28P3(c40mKHTeNQ)D>Kj8^x4L`m&wc^ z4hdYb{RzQz7H+=!FFrTFzaAPpZ^z^!e%{8EW9Y3q7A~8<40_4;YwlvQwJl_X(G~%t z0&?slYa{(-?hEHzq@A;g8EVMnDyY7aUe7?^Unw}B?j((;kxZlsjHxZm5ywuujpThX zol#7uUdx3+uw~`~$BU?!=!tvhHaXtii?vYRsA!m&d@|j@(r05ONq@M+qHtu)`?1y~ z8h*EV5UKUB)0>^z{|cf^h3-C=Yztj%QWz1Pl*zWx#cm?m1Aj_#=jDTFW@Ll+lQy2k z@nej1W(GBcf8N6FtliaxxObo=6h9AInKLfx-6QD(J2BLD3l-vvye8^h2CUw0&s*mb zm9q`UaIg>}o!3*}#*h~iDNLS2?T7I}dL)11?j@reMr}f{))RNP^@>{S~Tb_3(dEF9P6$C@iNgQ;W-I8n} z)b<-*pptX%=m#K3kRfA1_c?`HbDuL_qCh|qWQh1nkP&w`8qmCD_6%$*?aAp92sVjw zajv+qrqMK_k~7qqTGVt>Ad=Z#(Ijx`-qWa_b`Q{m1dy8^4{4%R*nV7b*fAqh_jnj( zWUj}dutT&m--Fa`gv`p%qFI>~R=fRd!xoI>9Z`fvD8f6bIef4vJx0v9Us>uq_GF}< zc=s$&H{1jy;dWi7_vbNFUDTDs)RlBQ*ht%lo_vx~b7J%bdkfZ3Ys}kSmRF(Hx2YwM z{Cj*J1qoXFvpknHiMalShfA%=rBOeke1q4rimFm0Fs)vc{vGcEe-5#fBZcRP7g5b| zoW%xx%|Tz)Eo(OfRdw(vFd$AWIRP-j#LZ1`3N0VC8l*Z31$=vjKJRX*rcw-{{vK*O zwgLfl9nCq(h9+3B_99m|u-@hXB7oIgZ;Vp?{IGvjP;8vvcE?s4tGsj1II4V7&jK{~XnIs*nsH%H@?`|0hvx(=Ks_<4OKY&K8#c zOxj7O^-)xdjUdA5Kx`xG$vU)?jqA`$)}fZHdrxl}W8FrX=Ijs6Xn? z9vXEC^+!9H*D7Z%4nrJ&hN5Z29=#JMGrGob)h$f2hNj%`#3*llMflO%K_VLLfk7zXA?~VywGK7IH?MIXUps7dH6m7o!G|kx1|4B1FH1V98 z=;W)p-I+b8v~*?-=O9jK;f_lWh?PAtAvxp;9>l`r0cPUJ8N{2J8}QnYLjIZtK+7i%n#*3OCn9-Cs`kbV}5Gw#JX zLjFilaEd$R18f&bZ`F@OBw;#te1{h}Vm3~dT{9m##oWKicB$l+1dFZT6KsnUy1oQ2 z0F$?<*71I=t#e?CDr6V<#8g>I0<+y?D|rw6k{Z^%SMY0(TdzR$T*FR#{8oP_V@&s) z>MBz+MY)oImrHC_ zjd2b!ShkM(o-8qGkG^D^>FH87B4%0>DuNyu3bztr>rtpJ%@z>9v?%i{$NPzT868fX zJ0Hy@55gGjxmdElOxSRG9@3NQ`71D#JnG$Gdpl3Q<(&63L$j@2+pnx)qH@I@@5X=4 zX1}Ix;*)sts7)%Hef08wGO|A`;ow=u1}Do{l$8|f(Nt+h7yC1+hY0iK^H+Wa#Z`8I z%J9N7`yxaEffrKiL+U~4EVLNc`fYype2`SO3rVux%zO>r1Sat3YZwOHhyBQX;G4NA z%6VrXV?9{{y<&aV2`!c@5o<_-2+bF%TQFa&LZ)Wn{8RZo9;xpTex+{oldji$RxEYk zbm76I5itwVdFJ2yfrum~K-kGRMD5I+)4`mxE->drit$^8s==tdBmFBFns`}kpa&3>Ly1yy(Y^xh=tgC-d9qD5wM)S&Re{WM6YWwT^ z+~)u^{d?~2t)oWd19j2R*kw7rF8=q+3oWg7I^UwDyR@5AQwlAyWpn7>09qiH{-avV z&q?Z>sq|tETZ|q*3s)mBI?Hz2HaG zB(>%n4A;BKKFKiwZO`9oUdASB9Q7TM|IBQ9;)*WLGhxkFNa7Wy1C0IJE!m_2vm?9OPq_2 z(ACG=$D~{$`bM;MJ!I$kc-bmkd_{I%iG^zl*t6>+$c_+q?D-G-u35(;S8%nzM+xGn%tDpgB(!XwLIAwJVylX68rI z92si)G-ow5=SflU@Q}i+L34ryWUX(y{cKXgze<$XCo*M#QCgL=NpGPk}OAqoxDOKwDwE549Z*kT$fvVvcI-f92QjGmVkIAm~D# z#wlh4N4&hrs&&<;M7LflNv>d`j`MAUNq~6T^o`LMx&> zaHMHn%Xs3xur$uj7oU1!q~&`XtDA>Jy_1`8f9sJ?WB3<~>DX0+bVkf63*xkU^t~y#h8)s3rO1Lzo?F8$vbU=@tAiQ>>|p7hp1TA z82x2?1nCLS>A?1gr43;RW1l8grhm=12%}`_;%hLbRN!lab~VsoNDmfxT86u1mG<(7 z&7Y0B=a{8SC}t*8Y-pLfG$NNCSDU_rW?=_%lA#wuZS5>GJQgv=busS)Nro;xMy3+` zo9~D(sV2mcgxogNOn+51YOTsl*BILmVu{k5cwgAq@ z{yadC(%fZNNsuH(3xOhhsYbY=BGe{xwak5(3(`)azmp|#gL0R=q4`v!1WC)LU*PV@ zwRv2c7(?w{KMT;K)D5f9I_(TQ+M4;;(Ysj62H?hY4TlBTct`aHB?4dyj~NW4&dnLYjJ19tLCYcp+|dZ8X4kLH2*zK* zKxmE&AvJ%GGY&>D<>{YFY~Im~6A-ChI{(eNW|FjTM@svKiJq129wY9}j9>;tF@kA6 zjGnT8)=1M2^V(K2vn8U=r9z?7X-nIYMPN!(K7CbPM7s+ZoT zd!hEbjmrh8J(n}wOYKYfv?2@UDk#^3q#pS3^@);+knJ`dJJPfCy8nqRRn}zLJ5Bpf+^U9>2r7hR6O*PfEK(1_BY6|HTPH4vI!Ruknmr; zI)z1vkY5WwvcLCAS1537PRyXVc_EV5aHdff;65Ic2stxb<~< z5LoQsk9(ms+Fe-dzl-Voa@NLi&Cte4w)d!EcSZ^#5ctwbRL`&7K z|BXfXU&Z74g=4Yc7(2RtLHlCehVJIxZ5SEZ+(DO1nnOC}+rgnObYnM$J=wYi5B-ui z4TMu80Oq=WPS{Q`F>gYXibPu1Z;L#cs&DU(+)FX2ktN13gUPavhCNorys^{k7o0wx z7)4sU$mXTslal7$5jid4B6GasU4gkzw;B3)vwDaKxtQlD1)5+!rbokDn*R}$jr!8! z6H!w;?gNuG$7cIuu|mO~dGjCn7(Arz7)m9G0ON;%L^A@5)(gT6bq&dTEAT7bE46}{u2t$85PMkuugNPF8rfehMIj6kvEw-C_MVc)W3C5 z3Un|cCFy5)C%3uLm`{2W5j{@zbubZc%#3im);_VfWp+Rl(=MaZNElP8J4!SCj_e@I z{2@`w#gbMq>QYl3l&<|2%H{#?MW2XE1n0~dF_14G4`84{;^fB+yAz`%sc){oNG5sWa#QGX$ zJd@Ty&Dd5R9j~VjNka^zTJ!ZWfA00RNFTE~MY$V!8w?Jv41G+tQu?N|)xUUh5`Fxh zuZ|g0OUaT`PrP0{hZS_rlg)fZYC7(4HxcqRDHtcaXd<;wSFiqdS!6Ai_QQ5X}Gf55fFW zOE6ahIi;%{2%jlIE*0GShl%it=3itU40aeP5o@qiYv^OC7B8|?qmFro9`g2kQd>438Vn%)4;=u!o3w*dUH{B&sl>XV`ph44t$U`ySa{Q7O2Uwg zTrdedY8L$1|F`I&I}XLjIoQKZDl*5LZ&=EmG~Mwo@7a8f3Ys|_PM0!=V~efgLD;Z# zV((yQh9162D7@HWFVr?z=q^VB;BWI5a_M&|i-%u|A4E-8$ujPjT!!u*M;CvwfXgMN z@RKVuyNz18q#u?r3PkX-3bS{=w~HPmtp~;Sal;C&52f|IiJVViTpLKs3Cmw*_A!*Z z`(+)MFqBI$GbA@zJx$UinZ745TDnDdyJ&fbv6G}EKW@0FBFnkiuqQKMPp~$e+JtNl z1){lH%aCy$1C-a4?^_+7jYXZ@UOA-C<_#0|dm*>&>CLbK{<4~VWw4|TVE7s=$=UP?gwu1y1|^{h*q!+5PYnTF z)n?Qo{TRn(U;jzQ9nqIqi7o}T@ zYM5zHF1Dh6U~Qi%`=c4gq$l#i5BZVr#YpSLP+r`}@Cz=63(WSdEB#sAg9`kTZ>%^} zfS$Jo9&N^oJ=?eAs!wRL-p_j(*}d0~bAfNH80FG8sWUW*$zyM0#Xac|44k;MULk{F z?~)x%Y--Cdc2xi?z@2sQi+%@Xub69cr}m1Q&+_dRUsDfvYOnaF-}ltUGGU{RJ{1=1 z6~{}6k{d&6Gm)Eodqto1-1St>*=AEId&Qmi6nn*6Rh#S;tJ}?9@nC*ZX9OKPn~r_d zhOKRIf7k{*G>lpm_kJf#N_X&5A6LgET-v^I zckXP;v~U;co-c$#ofMeAIJktx5Jfj?b(g~I<-MDK zho~6-VlzKjdr;Fu9yQVLIPwow7nB5lVdOU3FYV?_Umuh{#4kpm4<87tPWDS)@0VP} zn2(>KkP!PS`W_0!s)@mu2cW#l_hzX7RWw)sR5VlQF#(re!4m_a9wTJ_QWK(r9!{#u zn7QVfRZ&76Y@z39kZs-43m^#rRI-ndK%{P23QV^um}cR*-C{OYWNka|4UI-lW-c+u zx<|dqN#Tr|@tc;GHdk(1+V6fff(ry9$;|%z;SuK~1j$(pQy&uKWcE;2i8tGyLJYoK zV8AKy#u~wuZt4Ud{a1{sENy^?IZ2N=j9*==m4nawieY}lreWIfPd4vwQu|}(0kDHk4YEVRk zv$5e(K#b;^8Sd(6>xpRE-sTjeus3GiI5P7&8QNi$W~lHT>FSY22%e)DoJO95=hBiA2@gx^ zLN_f3JaARSYs~wT@R!Yg=bM8f;Uz8H&e}uoV3^_@L=kstAszQsn^M9Dc{kRmRE$-a z9Rr~>rJ{T2mMf-uXU>WXE9l8bIV(;kWTth(oE3|0nu;t<^Y%<~R$SRjxr?)6UpuC$ zn6E@#e`0KY$+(=20jrDYdpumXW!8bf(tZJ)yq1F#7UCtisnC}R1zYXxL0Yda-+UpS z{6Px1iMu^Pe6+NVuR6ikM!Z{lG$9~fr@_0Sh3iOmX??OG+c0cZJlSSz81(=?&hP7~ zq~3ip&?P?oGU3c?>SCc^ubPQk=Kaj2{2{5JVI;DYvOh>z1p*pLrHZvOniKQc<#75} zmglqMD*7qrkyry(11SVW0J>RQM%`Q``V#aHcJN6-Qzq2sfpvBYoqgJRSl73Qbneo% zdHI!fI6WC?E$pH6gPiRIvTVyza80PKkxA4l?G|bAqgEqHAZm3-gDjy(@zR1;rw;qY-&-hd-a6FEP|cB?pPR|LQ8@!~#c#7Q^0BaRb_H*B(Yk$V=vt2p z;apUWcUoV(n zQmx4VhYW~$u38gS8RR6bWo4Ld^6s=QJ`u?-ProLfFprVbUGT1c6VJ!J2}7lt-81S9 z7eEh?t(y`mpjjo2z2jlGCbaux+Q~ld411gh6+r*v+5*d2(FxVQrw3lFopE=iv8^^U`w5eg9+GkN*!+8w#9DI zvb&EU%m$g`#L~QPG%0I%Osm*q7!g4jW^^Q8H|-V4KlD_*rM>2y0jOX^*!n@g=IZ;o zLk8;f(hCigUMSRly2FfpVbw30;lgljGg_sObDC0OrYJmZ6Az z;>tvR*Ai8Q7^?Z3*xX;(CKj`~rjOqWd5hrDaYQZ&^zAmOyh8-lh3bjjR;0Gwr>mGir z4-EGNHKrEW6R8YPybHQ>a-oWSQv0(PLS-fYMMaF|O{42Be5cOG^z)*N>B@sGV)WqHvkMqTB)%&M&bmDm? zZ9}$T(ElMAH2IHh0y96}iT@Y+(Z?M38HLGfVIV|FWl!Sx)ljI_Nl|u~?YRMX8G5aUK;AsMZfSO(20jadmyi zdyM%Cadk&sZ32%%TwTQryb8aOorM_Oj$LTb7L%#^{kXcD<`q|B%hEemEU@&vl_`#^ zi;UdD>c*1uHSj`oT^+aDHxUINZ;ewiKPAZSSE0A#y|XtHW(+3t^P;}ssliV8I~~ot zjasV^yAQWqL4e)i^$RCdL|R_L$zW@lTr^f@_CZZDR1}h95$(XZ-7G&ed>gzG_0}t4 zyd+U}pUFkp-MmEw{V2O5YJ)Pc4)0{)wC7rs-P3oFvL1>9TXC4(l1J*(vqL|A zqo2<{pMZc_PLSa?^Mw>JF)) z-ozi%CPB%^-iFrhGJ-4y>o%^iSr&!bzC=6Bql}xX%I9jM+a~VJK9!5YjcjP0&}J?k zpFUS$W6W$HJ6TNQxLWDXge^RhSE0!3?q9>m=Fal}bCn%rVCM>em!yt87FaVoSG}3}8cGGNp<~vj zZ~R8qrt>jF1b)T6dI^M{b6^ATxbT1p>`zg-HzCum<|9%;c;{dr`D=qZ-z1;xRG&vr ziB0}Rl!&`;3mV$)w<-gheB$p?KaN)31L9?(;rBwV1Zku2y1aGdHw?88Apyav-ZuHn zWFMP+-!}4pW0PN(vlsgRyFGq6N5}tf?eVb_{`;r0$G^OseIeUO7V*~t=HdV57T*@x zMh52iW*dn;zLV^~8Cxck(P0~Tavytq4!E+7G<*EM@A9^hPs=tkwl^8DA~d=zu*Y}2 zA%r3}dwh1;PVDjF-%=-jMaGfmAiEt~Nl$G@=I!x$oVUk6;d}nB%Gu-duGk)5x3){; zAT@t`n#Cmc_zwe}=yL*d{C{2U*ONEL-$kHlD)a+Q{{|mF>*2?kU{?Z)n)e@r4Q?Cz#_iPx<1Q;~UBt;D53? zXO1tUlz7>zw#@&=l8h`gkT)_0h1ZAXe_tCnZ%gKQ2Vw9DuhOk~@!A{J5UfDnBcA-d z9XXHqz5MV>PB>A0*@J1FG=1v-9^b~iD=La2D2e*-B!Ut$a6f`l)H?(ZH{TTKblecP$^*9E-(!meMBm#eg2a}a*m?77VP>u|?AIx|b7-qd^mZ3n>@%=+s9vGCc| zeT}~c#=~=ofK zNcx)h^AN(m`V{I&zhnY~@;b8kF?FG~Z(z;?){sfwIkoB2xx2THiC_)n=iz6g5Jum% zpGTs&CE}9$n8wGHR_SR{dIs$45*7BX^LEq)Uu_ z0{d$|<>|w%oGqL0H{x@k?ZRvLjTqkZJ*skkBj$RO^BeIQ)ub4()Z!!Pr+1TmlH+9C zp6OMQai!^tl?2wS+Hk4n6W(Xjdzw!e#YO21mr_I42@P6Ma&t{do$)k9<}K6XUhz&> zl$UnWHdf7IT=|CRr%3%@C7@V$`C9(k!|w9Nj(+s3l!q5{W!a+k%F^BBVA%(zB8PKe zd)O94GBlK<#^)a{8NE|7#KT$&h+omSnD(8H(pNjdKzjISN-`E=&JVcerMT+a~HkK^wyHC#i%*+m+z-OLy_GZ6pSP3=$Bg>GJ9>XRsr z-ZR8Sm|hD6^6AxDj%Sf9V&UaEwO0CA9n8>uDXjy`it$c{BjVO9f3EHVIz|z$- znKKaH2f8joH%5~;`f^v~{F5D23oU$$0BgErHM1k`@*;)SA$A1!?$#8WRIu}{LE$^~ zLFk%I)b5;Tf>7z{25X3L^SdFJOW@zXHZjT+(JLvOevv<@C}a)xUPcSjW$%8&VKe zQ){hZuW0QHYAq!+h0QRpc|_aOO{;ab35?%WNlrBj!0CDUaj5MO-TGqSaFJ4LZCXAh zK+@UhO1nzYXb}c-Q&&GJ`_PwmAs$8wPtAT*3ttMK859Jyz~kg>G*jLv&U4<~J~Be< z@)AZUgi&R`kU3QH-^r<~Nl~sD=EyEahxG`;Z8|%I+l+cMuoETRX3U$vm|moQc#ISv z7N3G}o2J2Xyf0h6_FTA4{A`NFF1=~tPBd7*J_3NuYp|HeeIPQ3*nD|Hc-EF#2MK

%@iNw~l5p{ZxrTUGwHW9D2 zhha&lTZdYq_SY`yjQGOir`Myz>bmsede?Vzl=8~)&fGzH zHK+{TRoRZZtIMl_2NKl4cCnH)&^%-Xah3gkM7^I{x1_qVj^!TEse01xHVnfHiy1<7 zkxjQ!iR~%JI|h~mD{y8*5~KkwS1EyNQGk(9n@p!x;Fpz7s1R8Ioa9kobwaW2g;@)C zN)y#7Pqo2Ht%pb1>a>CI==ofw~bnQJri@Pyfb@_=q=%K{+eKI{%qY-++bWV)| z#7hy49?IbBisu8)R%FPc?D>GdPWn!?PmR((IWc!e zb?VwTeYjzZHllq}v<)YxeRAAm^4rtbJ~_qOCsRdftJvfZs09^iI$7O}^z&+cMn-+Bs|DljxotkI2Ioz6~Ss-nyrH?^YrH2r45w zJ3EW+4XcFqFkDyE-6*sN6JPabTw4OPmsSAMsyqD$p|-6QWG3z5Pje4nv4@mQKX2Sy z)%Suo?aR4O>yO{}c^o;;v3EI-EI7BFVNY+w5mxL?AM-wgCg}*{slJO&Y^5m2{@N(z zF+%oW2D8?LqZ^{$Sp}Wgqt38L&Y*=RM!o|`gdI3`8i@1hW6=OqS5 zHxrr#M=4W>X;RpNn@GAbfc;2--gI;xkW*hkPs+bxpMLb<^ZMrh#{TQ`|E>L3f~K)w)`4AP4~BHd&J#~vHZHN+GazRU&|!LSR_ zKs!$=;xcc+Rjie_E`DyaS@)OsGx#Vq_E9EeIASLmve+ zj(Il-??=P0D_B?x$IscRHx9>!B6Rm8f`!R|0%L5N4GAlkt! z>xMR-#W?9lI$tMP*biE`owd9A5cl?$iI62XqLn!(5eCX92&l~`i4fD}HI8>NhIhZS z7p-#?Eo^{Ij8r+P<5vjjE<;KmYG21mBmD}0a9Sl=7-)sD%31(lmHL|ZIR8(aQ6Jtf z5yDAjq8~G?A^bEM(2H`S4~39n zXugDYvSf=4>_QAaKKGi$G=DaEw{ay5Jb|)I>i} z*|%p0jxmux_>M751xm#8Yc^gIPd3-2Xyp*LLe#r>je0Ub7LeFpqABx_@Oi~m=z4|z zqIl|jztcfc=>Q1^Mo*LrR58pXTakRLf&f10lemGj#9!&NuS(De5}uH=cN}?HO)&JK z(s_T)jcgQ$tRSKLH{q{YiR=gNZ^=0nasPU+md*WHhkiJ09uQE|*4!$_V7|JTe;Bg=Gz%O47L3kIulL$`|E%0+xcfgV*6*ZogT&h_r9H@(8%I4 zvs2&Bn;@pzwfJ@pd{`9IkFWVlQOw79m-p=?kZ8fTa~!^%>zE<)?HnhH+5QT*JNR~v zlW(UA7JNH}>T)s24ftZ=4b2h3X+S$K9j)Sia?pr3U*&VOlN=_|?lDF?Bkq{mlykH? z|JPL%!CA5gk9nvkyh8a`&lN|D<|XPm)sk)(xsC$-wY-)X27_eWODTenqE0-!@+e&d zTtGN~baLQPx*#B&-@CJqM`>~nTdX`ih3bhDsvaQQBF=|GF!rG|q3e$`Xe+;q(uI;D zV0fA4qki6u&XunDUZx^>ELJY+=Y6+92Bk6cy-Z){s_?eR%k=vyC&#)XFVi{ZWeUO3 z#^gDkrYCh^ws8x$@#JM1#^>l5?$as|^zR*wWC!wpRU?JpJUc%MN)f+F6_qsafg(^> zQJ32N{L3w~MME2caWOw11m3A9f!*Ivz49l~&|>e@Qi?)u$uzEA=Vw1Dc&E;RE~P)m z#N?H7?Tcx!mv`!S_>g5kl|B}Gr>b1BcdG7lp2B)MOLP#Dv=Z zXxLy%$}}nbfkb9SD*R}!&2WytRXuoEr0+o+g4gPU>D#0h@yhQ$E0{L>HYk|>J}-AL zw)%8A&HK8v{^>xqm+YV)PQj&hGuxb}>K*AX%x0w|flI5IEaexZ5EI{$l@)^GSOt*3d0XTnCr%%xQb z371w;u40!~bL(8AjwoD9-klYQ6&hjC{eY$PZrYpOpVGN?vuXv6;M}^7%N_cXw7&Fp zZVmb}l)e<4Tm6StP0I3h+qtaanR(zILN^`h*T%P-M~ouIJ|B#o~e(+>&ra!7yZ9;c-`rYe&o0? zI?6e_7FM?g#xZ@Uah2uAPPWY#YMX9C+D<>6MF#^)PvXf(xxW4hRcRmB*9S+savzfQ0PXLn?mXW1crSIQOZi3VVMR4uW>5CAqC0VY-NK#&XTp=`U0;v3UOd2y z`xs=w^){!;7_kq>E#Y}H>Zx>S> zH#c-|&-uVMaOt~g=1TYWfn6drF>jeTU*8AT=YeTInbZbMo&WN#3hbhT!~70@t53%Z zPP#$Pj(2H^o~2$^4|nPVoA&#jdZ$NXn~pve7JOh|kr*;JhSa+&7`)}|$PZ-~J-r<1 zKJLWwfz3_M4paGC)g~Xv&IrXT}UuoykN z%m9UP{jM`WP3PoIunX~eQS@3Jm^M1ei5#}$(^Dr&c^1&u0@YPa`}~-5RF^DJ8VCev zDSwIX1JzZ>$Xzf!PjXG1)u!O@!S|W(W9c(Jb=8kWaP`)*K;6N28D)XeMa=sKG7viE zr-8zmz1%rn?2E)b#=nr?9qtKdUea+9Y1P;l?T{iCCa8-@M(KkHtvzfUD=Cz zD}|IGtC(E2&Y4k?qgY2DrV(S#oc!%#z87T**c<Jk!%W?N$;#u=lMLx8J_hy-0%td+l$B+^|yU^-~I+%N04mlN16nZ zO~t)m3Osh1Y-%|#1&9~!tWWVP0pLQ=or8lptY%sA`K~mpHk_6dW-s>njSChlLf?AF zvQAx#{0cca_G@muGFuhb?atcT#d=}Q)et0EK*(8GuXgMK`m9$`GAa@n?U;{`8pDy3TOCIaMcXol1lfS zLANk#!6W3Zlc=y74NELp{?6e_EI+^r6dy>K-?5kysq;S+M%zFw)#v05yNcw0iwq(z zRJ^u}ko`?<^qnvW{OnF?a!7s=i2K@}!j?)V%b~Une3F)r3s_pAyX9<(IW(u^2UVGO zS;dNm3f#p{X5mpN{qRh!_51g-WfX&^nldFDLY@8aNh6AQcy2}Wo-ywvIOUUu;%hcE zQ=uzj3RO7a6Px#>I`Ay7-EoyNbJ3O3?A||;YEtt*P=j+o{@La)(N)!5(t;iaTG`zR z^dnaPY)Arf2|xUMor!TJ6TJv!s}>*3O8K`5#I=ms4|-h(Mu1*i1Qwqg z=946r4>fF#ESOO%JvVjYwcznTFdA=}U5lh*1WRwWC~|d?2Fx4m`~+U7YNmQmhFN{l z`tR-J_NpCVrWB-+p=odSPz*mX`>LP<$J@*!>K=8HSr*9R;2oT74vIhXe68y$8ScDm1t~T zPgw1vN6{!6J6-|()x?NJ08T3T&ky4{^6;S%1+#c*MZ-w$rq-q&qc4kWoF~IR$#20M zN1QYUBI6_MqC;&ra}oEZS1ZXJqT3g6IWzT!0NyU*$vKsX;(WeF0f+<0t!1i!+>sv# zxiV9DnWFVxS%e)l_YlW@F@l{C#z`AOcZLvuhT6J#A8|W)eHHdo^uMB%{+FZ%eN8*@ zdE1HV5M?6S9_=yPL5b7SE?+v~gAcqJ+c|dS1a{?EAUQHLgA|Racgst7MTvKavSuY% zjTWaX=QmD`g?eDXa>l4OWQjR zZSN{z!va}2;Rtb3j~3Z$T>%RFPwHNvjsvYdxuTlYK1U#0Gzn;+)8hpJe(aP=WC}xV zDSh2vVfoBLWN$6BmaV(t6Ry}tE!47IEYx`uKW?EG`U^Glrd=-7*LcN3Ik^%$F4PE> z2$1KeT&O+s3)O!U^Q##l6;PuQfqQhkE}kCJujEm2`bT=(8ylzF^~^i^XE)LxWY2rY zyo;;ZJ)yQAv25x4w01OrH<-UTqBN)VX(aKJ%&n;yOR^~aKO)V-e%hETD5rEHvxk<3 zZdjUrP@mDn0c6->tL+2aj4a^7^xM4dEwgy;MXiJ(*vwNcR{=on9jXIFc{0fpR-9B# zL_p!Y-O>^(h!gNBDm-2?UYq*5?4su_(=(p~%e|GcK=Ymk?662BsRt3F=L_zPKC{6= zQcN=mi3mX;%rs{x+j=Af8(WqJ+)hrEhfsGzCD05)OQ4cn;c{}bB*{u=^v;TgCNNoI za;o1MU@{YEe;!O0_0EMHf>^E-_EI#)sa)U>0y1VA*EYX^gr_$ebCSwj_D&9s5oy9F zz>^s$XsqdD(3mvEZ7)!?!5dT)Y26AQi@C2wl2PzjkB`UjB^RZOxSatWqxZsNiIP

HurZbX zYcBDD{ckqL%EVyst&hPnM2Ggh?KvH1HCe?7jFn20ObbeTWsR`X^};~z`uvb|%DsYT za3qLBHURr|FeZ(*IMOr}FNjU11xfVgOlJK*;TnA8)c>v5)OY$*-%yfT+1PjLdy|lu z`Ml<)K9-+)D9DaeAIncYqO{A*q0rgXdgrics5fVW;&zcABh((hB`wYp)L~nVj15gCbO7YW&lQrTNBam~`YQ zwJ_X}YYa=OQ#GfEPf-SS89U+(M7Nt9?<%OdL)TX}XB4^t?p>Ta=EWcG4*OC)VKd9LD{MyP*+_mtKqWI=bqK~ymI>2WU2hJSVQcCb1$R^hFo^Wk z#4>P{Wc?C@INWi+vm>j@VQ31~N8MF17zV{UkuzCcFmC@ULMoI}31S+-4+P0)-1r~s zqLhBMXlP%Hs6Q+Kxr3i+K*4JqSuKYh;!=RG#h2xoGBaZHnX*^NFpEiQuxh7JrB>}? zQZyjX&!d2VJ0qZrCKg+thIBiP#=@7-V^T=b;|8BM!vq##lzS3AKEhtn;|8B8Bg#?( z^|N4pCCy*{Gz%l1YzQ&%d@{wFM5dgZ-uo>On}3~~h+sZiF6Z=7HmU=GSERaY+9JCj zV1$E5r!t7An%@EJTE8c1+|gy{9p)MCk^hWtI6eGOLr-#t|W55PE2M5+!OQ0 zR}ych0dK?l1z(ppQ9Z(`z+Z`~-SV#y-;bZk{PcyN!j`&F=iS6;;tyco!sSIH#o$NX zf9UiZP0nJ<-l^wMi(M$T#{o$d^`!XZ)UueUr*}AsJ+z%d?O%k5QePwS&7sOpTuG3Z z2usU)aUK^05`fes;A%ECzr^ci@({n?o1(@!@Dv>kJ}pHxjPx?|^ilj~mu53K7%)OR zri-Ay3j59^Ma)0e+uso74T9(5Bvp;i zX7MgeBa|tj*Pnv0MhM11PKvn)f(4QjvKY&8Fbj8u+HR*Rw@d8ItHraQASz>VlWte` zQCThchq6fKig@^9LsALb@<>8J&i>{f2OD#^@K?gNye}m2H$NV~{!||h{L--4vX7IC zVB`T6^|yQKOBi#Av;WkP=uLrAB-qLp$pJFT?vgvIkx^|Ewq3AO(i7j}^W;>` zd%Y-KUBOy$m{gi`6RVSO13)!#2Nfx8h7Y| zeiEx8=V>Uf;yQ=mTC{+D@tIL!x>e3a1sImyRheC1@L%+z71@>PY3sLV5kZ0{#V(9g zTrCfKZJmyOL#0mggb@8SJK<3skzlxYgwPUBlxa3c4I;9T!d98nF|=-fGaquqDSId}V>(v66 z=y`|_mG&#S!bV;-9wQp-GpG<9z)_?;^L@UOb0J4_kW<<~>Y)x`uYW)&IC8Cn;m*&2 zLas0eU*Em7@8B(V)svZ*VJA7aUc?3^FA;-AtS0n@vpk!4Khn@J(`;}-AcEi&*GQ&PEiPg_ zhMV4(aR(D)+p8q?%!%8;hs&De``aeCD&AkY`3p{(aKp?Y!>^8>Cq@F>-vNkE`@jy-)GFsPeKKaQh<0~tpZN#1 z@N(>(=(XL&ywUQAV`j5B(1y1qWKf6Dk`rBKzo^$=7VD0;U)6?My-pjb>34NFGbmNZC zdrpz>=AdCK|IQaE;$6F@WK=njL-xtjkq0lUJ}ZktZSb#%w0eJo-`opatGV$smsybm zi^v(q;KCO@!>0Y=(Xt1B%=Rslnq4JPFxAyYrbNyLl%F zq5L)frukia+lwFU6`!~Vh;fo5CVzfgHg?`Sarg1iUie>=>X|yu0}Y7z8?|h__u$5+ zbC@jx!-)+AtU_%~06D?7v!T5ntZCulw9x7YYb40ggT=WAci4j~Ye-bM8nt&~ZS zaChhPNO^39P1#QPoW?=XoC?bS~Q zBtnj}^`=*vYs0nv1de*l2@e*2NbDD?l*jwLWm<}YGj&&L$a;+N%C9CU9;k4i{1Pzr z?Mfb=xeMy@ZQceL!Z37znHz${YODLC`g44|vb4-d(}=>y1=0k1G15fVwT0#7)_7uE zYRhl7AuxnE-n?`{)dqsl!ClB+Vbu}+N>-HriwC9J0qm~9Yq-gKiM0TRn4q})WLK>y zzn^!+&hvpO{bg-%;MQMNKf@UnEW!s5dlk2QIHi5%5T#c4DK#@!YFkTDsXK~F4WzCn zE47Hq{!Ot_S--j?IT10wx&j+@#y-x}6be%^r7ID1Kfx+WA`llf9tJ z-i5H<*+l`F_1oDG9bM-(5#6lhURfZHdt8x zm#gy7oS%Spgu&U&13>dndAloUW(V&An%_fz$j9dvKHeELk6c)Oj5Q84f9Y36(EK@` z2GHDZgrGUZgFG}33 z`jq-juGA65rPi|&xw-_+H&`h_^H7TQhUSqzG~Z*Nd~#?W89+09aE`w144y|HQUuQv zChZ8%*9ObEGSns^>Hlqb_DaDnb)ewivqVC{M>!;#&jF_&up2xKF8-qdW&o#Btb@`4 z?@MX}$FcxUubH?DIQ{uQ1*e1NdP;B> z8<)EJ7e#RTJ)Q<|dghk|r{Cg19!|f>O&(4Mf|bF#`|u+;t>+zZ>W?G`r}N2*l!L9^ zfvp2DQZ}sn*DH$Pw5d<255E+Yx~90)aYdyBr>|Qn!RZW&?G#Q!_Q`(*r@i3QxRKJ!+-PXHNzwFh$`jh1 zEI(5a@P+v8aXUK~8AheWdVspcdQ^&+01Xe-FNhihhR@?PCv@>DRZyp^N5q7PWy)oB zP+e--I}~Q-7=pq+%TmkJad@Z_T%Y4Nh~^NhX>4$}1aJ-?gni{+BnG>@lCZE~XK^+@ zM*bOAfQFGZRmB+jFV6|mKS!87pT8)A^mW3>_X4Z~dAlo&3{BVtM*gPkPm(7*PUO8l z#}}p8EaESIO9_MYTYg#u>5F(8K>Ca?3exBBAP?!YxyeI%^vxKa+PmK>AOI z_lES5aCrvlkIpWF^jUpM?VT(2v*J>XMWqDkC00t1KA&Q}AwA$PK5w6Va!CJZ{vv?) zCHot^$Bn;u02`7b{$eNat~fiN0N%wxA6^8;GQ2S#$mW{S0%4et0?_+=M!KKli$2Lg zzT&mRi5uH3z~@u{a3F}kMZk%T+G72}cH0&a+r6jHc3*33w=92F8sZ=AcqhyS5_Fn9 ztnWr(i^6y}5eFW&sTaSUeYW>b_`xZ=*a?4Ft(`FRvYqS%NS%OL>sr2^Vvd-wFF3YCGW&ezX%BdAGw( zxS_W9PWZy-gPqWsDB1}Z_9<17EA@loQsatBX(wzVNM3f8cEXhu+v!d?&_4OJJ7Lk+ zBBtO%#5v;5eFgSg!R6b0_IYLV=M6&kh0VV3^ENuFPAmH(9^_Z{MsD&edpwX2HE-ZYD?5vKT3MR`bIfM!_kN-G%EmAP zHXQfbr9~@yL!VNI=StmIToM8h+SYVu-qp=UaQCN0(rfO&FRAOvu5rDd8Kp6Ybh(0P$N75s2T-gFJ}e$xR-_zYcN(#J|apK>P>1 z+W~n!Gpsj=UjS!sAl{$S`GeK%eM(Kpm0CTcsFYh&N+5o$l@f?QOtAup7l{eZu}^jY z@lPNoKp!5N5=0IESK(ZYYAWsXPr}tl?n6*l9e-!&sb~JdADptijlJegtDj@J=vh zBgq|#zRSzvLs9u>0vKC-aS`-P=u_%7%#95aXBLx?gr(*CGypG+0oR zApt+Me;IT2{DnNGKM(Lr4LN-W?<4m9xQqPv2l89=-t22?S+qBK9;~>9<_%U3RuR>~ z+C0H@%GOGm5r5lO`rg7vxs_I$4;>0xPWX{pY~FK4`S*1J4Ax?gf+FQ|ZUYruXFsXv z{;5M%a`4ZvWzJUZAs8Yh0y3+Eu)w7Ja|bFvB@R3U*)D7O>X$=p?R>DX{AlWd7LQNW zl2bJ=&#dODOvr!hPu_;kr*wFW{R~#W zolwK7RO-U*sQ(sHx-|V3SUmN`@5&aTyp|`KPFA@4Grb}8EWR`t-1MCyNd0A>QYYj} z^%R%7qED&exl)RUM`&>$xnl9*RZX^{g3za=3NEi6Y^s!@zRZ+#4Llpfv%2kAiQG8U zm!5VZJ`4EDkucb>9AT2NveML@hsqLabSd_u=W3L9V*HoJ$~T+329N6W0bEU6 zg!W`K`=7oT8+FljMN2o8r|f@%7?wBeZ~LDI`DKf9lV7%t;0Km%J3mYmmO4_Cv~X9v z;fK&Qo5tI}T{Mkn_9?Y#|De>o;!;NymC}ZP#Y$JLIU{l&Q%Ae8|{*;2ZRTqqAt(h{VF%ihZ zLe&-TSJaK+RWdBf>C^GIA%+BfqLn>T2hS3G?!&EV5U|4towUj_J_KQj{qJw?^*yWj z>IHq&{ClQ8_j?(g5qNm3XG%CftTF2cwy656%Cig2%LQp(xk?fYnc_eG9hu7W_(%G@ zI^Ujw)UPO3D;N|e3bUyI4jvgjPm@!trQ|)~0v_&!>Wsxk37qis&QZ})Y&iIKMB&@M zs0h17(8U025t^g%~Ee zK@Q0FWN~g8O*7u0;}mWsS+$V6?Okvr$6>3`UD@(M_{v8$BkA4gx_=&<_eP7brTeV- z829mLcx&jI*I3! zFs1!w5~Pkc!jCn7$9*nxpDK?${Zji{`QnG_pD<<0%G8-p!-7#R+ZDY(*RuOu)a_3q z+2E7-ut(zI$7kNE4Tfw%Z9{NpD^S-AonWS;;%K3Eoin@ctK-UIm(xJ%!>6>bmy+O# zNs!cK3~bxL4fc1P{as~$JEE;=k`U98O4}FAN~rxtYSPIB?ea#kaMcm)&7p1XTQcs1 zxaA0|QiAz}^}W}aBoDVT@<_2>(9&l!X&wxf3G3UA`?{m5-l}&$Q17vmpjI`CN+;%` zJ6%WoJnT14tH5mi&b^}HWuZ3Q{TRZNsqZtCMd?3t#|H{Q8frV2l@h;Kp4hk4MkmG< zy%6fvP-S^tm7#V-GJ9cpi@cM9$LkrUv)QV>(*laBwTA+12rxl{CZSwI`6y#T6p`%?AwJTOY58JRsPP zh8}vn0D`TZ+n#>8ePzV`SH%50<6Rpc_B!b{`pqO=;UuhBrHX6Ys#dr?C3*b&7{-7b zn#V?mzbSi;(Cv@M-Tye7{}mtpj8oPV3qKtVjd|WF%K!t}XU8WG8y+Qi@zn`dkRdq* zSulaSW8pWOFG0U@`w$@piz&DjjY-_wFM!Rbh&%07E$;oA4rH!2Rla=O{e*R7Ub2LB zfeFy0S7qlJ0(KJQ4*9Th?wZW&+DaoccL2_Q+`81tziUHQjSp&P3bzWZ&@B=Skl(1g zTY5jb-Fu_`fUnqS_vZE)K-h1C{;boVRr=H6w5IA3|7OB+do1AdggI*C?ps0U|HU4& z>{#SF+VxIb_= zUY#PAB(-f}r7;yNJ|Faf-_f?fn6qJjXf@#+2)cM!km|HPQQ_pbl@oesQ{1g>U9ru5 z$9?zd*COtRv0>|ERx@)iv)f8uTuIPZ$m^&&xVF!2{Mrk4TJirN! z_cO%a&k%b*L+t%P>;T<*rX&Y+9la3yzYMV}@Vxt4)O}-@i2XoDL2j7d;KqYIxW(L8 z@t6V33W!~e5?8Y#<{}DDk=_YRH$DetEX~va#_o9x7{hUsh+gA>F^*K{3dTA>a|Lll zZT*4{GEqd`Ckea|?Pevk7EjGO%gGaL%pY?HSS$)NPY)AlFByi}Ci1Ylk*tYO}$Ft03@NuuGt3P(rq-N1&{&vxqD(C&}X zNF0P~)WbG1tPKyuiZs71;^Dp_*sqQScvj7UXXD#X`i{^ZGx+y45(F*F=Xu8!hXO zhHhUTEdyEM%eOl29$ynQaoHbKOyQ-Ds^Mf#7~A<-vET3^tC-|CyjMlc`>6%2fbjwYfi^)TFOIvfn)DD#?mixQSm0LgZpEX9eKv?xwD5j^q6H%B z#)f|o3vZ2u#%{$U_%bEu7~yPQ>y*6^4Zo=J0Nsmh@RwrVDM06%3Eq&GvVanw!J?c^ zOC(e{SOs&1oU%u;L{&Mb>}iTl&MuD_{UIXr|6%W4z@sY8Kj6(}Azb1KN;F>3wS+b; zYExtT2LauMT{(-pnjk2mHrx`SBB5b}+JYqAL^(OEq6IHilzOSXSj8IxqGl5y+|&S7 zxmXPsWtRnw*h09-e!t($oXrM;_3!sQ|9;PhyU&?-=AD^$X5ROm>zSF8XlY#RaQ0#p z>+QY8JK$q2i2c?0WWZ6Ot$t`L3Kq!Z`PeHV#i$cTd!?Pw`0bT8c}Q+4ZN7^`1DxA} z(#o&&cC54$u!DpLk&aVZF@ENFoL}hWY(38{l{TM&F%rjh)op`=!o+}_gM&g`QjGFY zX^*Xf(sK7QVzC-vA0`~wJ_QAJvW;;SyMD>Nr)+E3%|XF*d6a5AqH^@%5WF|iw?}_U zedc^SA9J|&-1-Frf6Kv@$ExD!=N?d19Ah?Aamg*QSKg}My7-#0tl#~&wd>)~QKNn* zdN@{U?vBT6G}b{3qCDVJDyEoOoQ~+hm1w2WfY&9uI1J%&#Rd1g;SFGP(@GS9Hq3Z& zTLU|}apcwOY9OURQhz5h`p%^rG0_OwKWgFH{3_RM3X6R^_=Wzs@M%%^JbewRpxI+1|#havX@w@9y0(M9iyWSHh8XN9U!?K@1-eh&$k5;~qO7 z4ksMxbo5wO2_T|)2Yjdpx1cy258yLi>d>uz@Jq3HvdoF;7L*0s?6V>RS8z~tRfa&S zT?1JhS7&itoyBo=R-GQ)gK;(X;iKv0a4wH(Cx@A`KXhPn_lGaU{{ z)gk9u-T~gMS-HMYPp3Dyqw}?)^Bx=)nzPl$W)k0(HR>WSBMuVKp&zFUK6npq^`$lW z%^Pu=Zqp1ovQc;QLp;y}yljC=P_Hqi6rHn^s~?zS%L5?Z_*%}(#Q~6;H?tiWX#-Jz z+6U;81W}X-YW_~_6H#ILwN;LKFc6YsI38+7B7V%xq4NErSD+x)#ynMi9t>h>ts@-> zjGW+95msV?yIXNGVU*W6#y-s({yYtbGtk~+Tv4KXLxWMJdA{!-SXIkI-H#U>@bzw~ z=;bx9b)ctNivzG(k>wBtWW^COuNJ|njN5bc&`?;ddd~HN0UrkXV>!v88%HqL_2TRR zT6Hn^6C8+h$$#{Rf&&P=NqnPQtxO;iuU+TSPs#PrOouvQ!UHELi*>?epiJQ;%XmjV zPy`C<+O6N=cm!6f96a#FG1xD41466F(vuD-VB#v~nEUzF5A52{n(H>c6zZgJYSz1; z+@diCfsVfzi1u#Cy%{UE%?}PTNgf@RbZH}A+s#qC)M^zRp zp^o)sHTa7WB~Ag&gks*#3jp#R`B{wCvZo!fZ_-k z2G)M`;264qs%ZJJG6HH|vj^vYx`7J3G0;b>0kJBl;j9;qo^yrS$LfNEalsXZ#nG>q z6ORauPRWsXRH#G{Fu^K6&+k|-)A zU>jFQZ*3Rnb$IRR655NgllV$O2-$sb)WefnWRXq|K z9;l`2mo|%&{K6FC1DUn${J6KC;h-c5o{u0cIj6GL}E<)?`DeIvxG~(~Jt3z`>6q$Yu zSIeOJdCNW{-Fi#Lf>#Yw@n9QJ#KnMCHYAX(5cr7dTyvjtjL4qrX1@!;7^Lfb*B$;&Ywj8q-&Y#hDR3kZH6+Sep}Pg2me`3{vr_ zw}q-~zjBBp*QYIG8N`V4mA}IRUB9J4jMruP*)xkHCAWYg@1zrN%Fyp2PB(qP>Og?6 z0I~&@$gfGEF%1M!RS*KzX{6whDq3X43kZNWxW-5>@&pPPO8IG7*bAMiW$9U3k6~F_ zR-r@dG2EeLjmp=0jLz4x@O6kDiWjOsAs*W6O_MU(ftw(_N=6s}6*-2P?xrj~{=JYAGVRwYA1F8$IAx7N zJ8GtR57uG9HXi5=3*UWIbjKBDV(i|cchSU#tHOA8#=EGD8^mQ5LW8-2j;30iOY&iA zm&ey+`JCjVAsvA?ELi{OCz>a&r#r>6GaIV&#K%`T7qxaaic#AE@xfzcbIDOz0GUlu zj|(}WY&*{=7Ch{aSmDIcM||}%~{NPJ8Q`F6x63BV%d zdqY#x=|YLNh?4nO9#Jy+f(1^L5rs+=Pp4By@hW&&@ZU#J1kN2P2yAo0`Og)FH;Fx= zg*P>*^0y?)FM2Gov-6qtTt}fmm^H1M-b_^RW4O-^9;U%{FR=BNsmr!2af;WbVo&tW1c0 zSDxop`fmkJ=zlt?uu(|pzjYreXiu_&V{p%HCcOL#S3Ou?x#DG{V!sOpdMT4WgmEOnBAWZ-H0ttbwx>vJ}Vz_Mt%XQd&}Q0~3i4Kn;1a-FS@o=dy_&tjqdJ zE6XU=0;_TGat*rTz1s3Mdf&BrS{RL^H6CSn#no_MSMP+!ySK2IS0k&6NjAFpX)q$^EI35YQSu7y}cR}DVO4o$I zP)v*)HfhT@#_JE%Yk^@0>cy=%_&!QF!*85}Xc-6U(*viWge;avpwjV8@IaPZd!4vc ze5#V+QV-Op1njjvT(eIHulqH~42Z;qyA-8!ts0EwCHm4OELL#KE#yM}x(dPJeDO|a zyXM8a|Gpbo01zzeDvImYJ`B4yMR2u~R&zPHN?x#-41r^_qKvB{!5Aet9ztC0OWT08 zBK=@JYfa$1T3pXj@0#7U8Y{B6j0c~RtXF;&yG$HOs84qp>nPMv6@~06*sP>l@l%)4 z6-hx-^+Hlb-xxfwfhjzV6b^=^61*hsLJ8(8yDeVv7mx*0@?E3IJ(PjMwKTEfr&Wjz-)UNCt_Czr>E7&-t&Y;$2|p`T)o!@x+KsIb184}N)AJZ_jDY0=6ZYIFNM6dveOh3!kZ#^ zJA4fm99c;dSadz8q;*W9Z=;^pf{*5+nU37Uxj4di5shYE@h`C>2VMPPyjzR>l73hL z1yWnyqK`iiTYEx#5bcjZYCOdJMTZV*@80ci@4mORy}Pl!{|W8ei+YFc+ao}qqA&0}lQCRcDLs?!F4-*wp8e#XwDSUARljHp*BgSExgy_&Ri zjLNI+-Fv0ku8236YST7ru`vXv#cm|Xj14E~9=n>LTg*++H8zN#OY9c}8L^)dbdKc_ zbc$U_V2?pbrfp7-ok5Tm>rRjwvlE~ND{E2;k9z)LJtWsvtd>Wy^UtO&UmZN8Y4?5% zJF!zg_?2C=}VQLlfW{FDC7qeT@K_OOvT-vi~ zAKsqTyolECcpq$xK&{xPkKgC*8^zQ+uLTPR(@(?Kqa){F=>^)4-I#J6#ihMFr;eZih^ZUF%)nNp*Z{fo=91VY-Z*xG)?zUM2}FS+oVK~P@>zUM2}FS+oVK~P@>zUM2}FS z+oVK~P@>zUM2}FS+oVK~P@>zUM2}FS+oVK~P@>zUM7JomY~s@8ztO@*c~n=5COVPU z3-G3>HAOeW6=}13GQy_q#}d=r4Hy(~TvUOVBfS>H(Q;I^R>G}?p2O7B zo3?sLJhuTO6gfTvy^Om*K*$1w^qJ@uw|Dj)#4IPA;Z0lP?;A$wb)}^ck_>MY9gRI_ z6djpe6puY=6df)#ijKt|GKvnt@->mQy79m)44$yYb1$Cz@jQ&@Nj%Ttc@fX+c;3c? z-AqMT-zmb&qoPmn?8fs2o`ax+Cb*~)&ulz%@jQZOA)Y_uS%hZ^o;p0M@nEc7v>8t` zo(P^6Jgp$`z}+$*_rQH0o`>)}f#*+nUcj>$&s%uV2^3*Pz6b}Ni$2El8J^Ga9AM!{ zXQrf616&Btd^`*AJd5Y=cwWQvCZ3ge-ob-SfkoJJR`e;JeR%$@ctcV%B&lkk=ir%# z=W#sG;CUX;t9X{kFz-p1^O(dszUIMknj2lr%LFmgu^wL#zGp4 zXw=bYqR~vFr2%HuwJKt)MH+9@c#p;>G`?VUgQbca5ea{(;1mT} zjTrDg8c)!8fyP@j-lg#|jn74Gfvw7~;@zSk*I0;WHDbsRjRiFRPUB4)@6bSBFw3>i zsx5F-@!l}Pn-!d_Agd7r&!O=+jpu1Br?H;KM>O`bw!l)w?S6zeDR`TLzg1*RhJK93 z-)Ou+V;v3jYqMN2Ra;=M8m40MWE|bUQjpb%VQXppfyQ5HETgfO2Ku{Mu02+5fuqW+ zqVjG;x+f{fYQ(?>jYnzxg~n1EYiOXKnB|Hl)fTv`bQOD)g1D(&P`y)8o6Sg%(D*Zr zB{WvkKo3646~WaOu2#XTp(^4H3f`*V9g5Js4F52V=V-i6qmjmkGYiJz`$-NPsk(fAXM#WYsY_<+V|q$L|+IIBD=@<;_s6y%m?2CtzppT@H^UZb&+ z#``orB{8e=i-K~B51*;EEn77q-Lx# zj!?Y8L$<(6C(!p+MHsqN+l?f4Gl|_yVz(=w259dl?cJok8?>v!46Rb1S+Bj&id06m zr7{x-A_AL?m`^u9gUCf~1$g%ObCYcy>RcT1U>=33?m}9(E^^6D+5(r$SFc%85PMFP#{LE!hcC*L@ z<~2SeeDtfoMt_PW6pOKHzU`UaHOi*WLSu=kYm`lw@~7J~dsu3eH5B7#diP5T54YR6 z_GmO$;GEc+)is)vLmojp>c2QV0y^A_P=+1g`hOLRMsbNy*ec@)kj358bfdqtw`xu3VUB6gYpldL=0ux+;39i6w_8P5t>39K%;&7L( zO^hsm&S`CIiFOl?gnR`*4rVd{Yl1DokDUT1__0}7<@ny>#|nXy{8*CYZ}H<9nLqU$l=e{e|fi=9p+t zgrHhyf;;?WDahj^{Ix*i_-j_M=_IgD;TL^JxKPEL`(J`qd9CuFtlU;PkB3&dt#bTt zp;himRsR2Lc!`qtsR}PsTixt`$Xftl4EE+#DKX|s*nI8&by_EsIP1V?A$D^7LZw0g`yH}aDT z*}p}D>&N-}gWJ5p2Ane3d1c^qe_tGRK&S?U+8W>CPisJI8v<%`_=Qp5fb5^Ar`o1n zH*UiW{l_+8I#VP_l_1R@s$>Lj)gA(gJSxH05*(8tu1L>(Ig0&k2su;VElrYy!wHfE zCy*o(Bnd#003-?8NMgMxcBf}{MzlZX=a@QYMDvx!Gh0uM^4K1c=St`{^rKM?c0X{P zCB*0LTch}j9j5tr_OM02v}{;1aJEG8nHtXj;j!7WVZ!Tdj;?`IGTNV3kBNUBINbw# zQ6$@C$qz|zSb`%GoPezqyh5dmZ>NagsRrMIKJ&LqL9wYC z?nTP|w1SVF1m3Uk_kKsn=8r#IQ~6(k<5aq%PEojBmHT@r_3u>ULqy5{dno09s__Ab z(8ce2cxv(iuV2OQdw6Q{;cF%T@1d0csm2G~Wi5W+!&8$F-IV;lhf@Bh8Xw*|U6y;5 zf=d-#q~OyEK6Vm#zrx@99pP_PykGy9;5e1;s8bYrRk^SD9*T-^s__9wg|P3^omdZ^ z`@j70{RfYa;#!@rIPsU*XP7xBi(;jZiC#WOrQhpPl} zzn1nCPg&zWfLtE5#_8&df;jz-qrA;HHsA^#OVO$iz$VBd?EA(CRiD~GGWb~v?uW(| zHQ{}kJpEgVQ*40>Ts?X{u3q%)C(4#mk>|pdM^!sucP`;!pMo{oQ{M2NOcyTOOxw@H zsK(alXYg?01~1&)i<6M~B5EowCeb{fi>T?Ys-1khf)kYCJw2-s9QSi|+tUZpQgQ2< zw`y%Atn#Kiq`k+xA?EEC14Z0x%c#hkw`x5iAq+Bt!$+RIdie9MdbiJwV<13^HYdo~ zP*4WRVvs%FaI72XfCytd=!HzE*pY5}w@9K$C=LpuFM~IeL21_M-QIVJk~mO~JxDH7 z@AhxE(WDz+^~t~@xEptJSK&=ihHa2^Ho394Rdat8{4~7}iBxU3AwPQf(*j8sg+-}x z-sf|@3KwuVY#`6$C1jJ1gPr_25x$u@(~X#*Zp#vx0KYU(Ob>5|Uo?@k6y}VAt<;_h zZg;?2gEy}Q1p>Lnfo~MpZESWM+n7op8^TuYtVBZkQYI$^5pM3IcU!5yDG|tVw+Igv zGcTHFr5hJ|hId{Jonc|RZPvTt{P7C17`|IjIz)~w!P`5b8dl+KsRtUc0on68rD4m^ zs&^q}=uS$L9(*Uos1NSw6g=EXt9b`#5hN>slV$Hf<7|pqPI1+*D5@w0i0uJKAz%~} zY2qNy9x@z4uT(S~xPuv?z=_~zIS6&S2qpQL&Sb!C1c(KXG&1_^~9DU;z4&b&&Q%FIuIP!0(EPE zq^furuM%pf1Xu~DH@u^Zw_ClGBNTu!E%`z>^ze@J^lnW$?wrIA{(851h!mVv@5$!PXxJ?7`L_RoHcH*}e%RlS9V`q15SNH5L4cHpLWdpXxN9@bDcDUv=P-XB^$N=ajn5;VEu&SekqG*`;oy2+HNfRZo%5*b$GXzH{ua z2y!|?q{6!gd~sPqq!WDc)OU)_0S$PjAUga~;D^tkgz?ndqdWuUHnG>di#R2Q3le}% zkMdwlB4w2-0;C}xp87QL2|y;w1mTei4?Oj$(eq?Xq~(pT=51E-`e+7l4d4Lz9yr}y zd%Alz52i6EDTtH;FX~;Or$v#}B-Q#Uz9;W-&#ABAW=79{n58rE`5|Esv}D z3_}3(4?J(=GaePdtmQs-_z7S>$c_0v;{gH8S{`eIp8)2AcS=72%wd&2!&`}apl17U z{3fd2ycgTNnj*A9!jXX5}i zipb-A$eaLXEj14=0nFN8N|yj;EstrzPXM!)#{l6Hz^vubEw}_QYwwmW0nB0dAPe^( zEB6832lx=+Lx4{JJ^}bAz&`=L0QdsnV!*|KZvnmqh${!ZxGrnhyMVZvZx}AH_I?2P zG2q95p8v3;MO2 zCpJ-7o{mJu1TbrPjtwpW%vzq9gi8RkmS@}G62PqGDN48mFl%|<4K4x9TAr+gO8|3N zH4?8zCguRn0h|Xo5Aboo#{r)Kd0>noEyxRbG0`3Hi0mcBo0{jX^JVJ4zm`5yW@@Oft zC4dP(;S#{C<#AK^31HUpI3QdCn6*5D3YP$8EsqGoC4gDWW2tZnU=G9AT)e+S7HR=& z0Urc>5bzIxe*k<6@F~E*0{#{7Wx$sKmjNyVtOu+I#D%QhwSf4{su!2G4#U@UyjuZx z0PXmo0dWA%8wSMv`rb8w8vyZP#$mW6&5KLxhHVGLE%?K5N4Yl&_$A<%DC{8?7Ik$r zg$oDnMR3o)kteJ2z3ucD zmj-Fo_rT!`Wk2~Yhb&vM$xg$yHmNv1$!;V z)d9y=VsT_x+(HuLanrMTfD;!gdL1)gkKTmQL+M;(0$~$0gY}HL1zDB!aPvhB$CsyV zB}UL9CW%%s%wv(H*XN) zJ??Zm8NCdP9xRd`F@hd39n$Oj4fK!+sV~Tuq(_XPM@)zGwqewJviyNbk{;M5=@BF7 z5z`?(^Bd?vh>{+}D(Mj;=n>N)y#e1q4@xHKLG2_xVgx;6I;0mv%W<;uq7q1Ys1}kQ zF@hd39nyR38|a~uN_wdlJz@ksVmhK1yGFDx9k&N)RwO+%DKdY=2ztbHL=P=ma|7mc zX`2$WxOyzp@gO!@E+-FcYdOEG1e4J;4z7bStO+fNljp$eV0dwfUD@zv7^7R@ENX^v zW6R99L3cGyK%cCo#13H9SXP5S;J9da%Z8!N6o=fsSxC7O*x?Qsqw|3-f^F7~u$2vV z!WdSFIq087RC)N_Tgc#J7#xS+AFPyKML^8DF>xCHVdSf&p5;|jT=0bLHM)p>ABZku zR)FW#aX&7``r^F0&**|7e+H%iS?G>(Fb&8;mn4p_BZvd|Jor0+&x3zHa5!#0+=(#x zz{?};i;(PHOg7XLml=y=?e*=>_d}je#b+k1HFRO}fGT0F0jV-|t-+fkl7*K{8(uPD zc!|^*OJspUq^xHC5$uk#|E3Hp!4Ov^1vwWsFRp$o8@>>>qJ{&Oqpjb~~?u*p|FcbpCD5fj;g>Dl!h1TOf`bb9fx2@I`o%3Cw(}7NxllG1< zxYQ5f7$)jORJpUB>%~|QR=1F2by|G>Xr(&Ten*xslP!mkM5UDiF9ChqwoxH%U`rtdDBkUNIQ?fnQabJj@s%$#)# zJZ8=+0sii?Ik{9z+UDghRJO-+>y+)8+-6~$ISae=nX@^n_dCRM?b?={N*d? zP|&7e3+g$4$HOLtultU0p^8`eUxHTpi&Xk^kITmx<=D*vep532H?s<5kS-%nN8OO^jrg;sxms=|fcWxu;f!MO@nDmX^L z!jr&!g?D^MXjAc;aqA#||JU$Lb>BgqO8-=YxvKmNRryafc&RG?R8{^{4O;h`8)_q&n`YG=FY5PVzzBhLKeLu&$`Mv^Q zDCc{MMl6T!!}Tjzp07ktKJyHIOcbx5j7|Qu&1#(%LDpi%;m5~u#ES~Nw%KuI;5_lV z0)f|xz}D*uf7)TZ!Qg5?7Ui`$b}SKKdH<5-a)(0@+lp=DzC5OvrD{13+OQo$++fpL zpa!P^=d^G`QzViBD-+06iA)2sE`h`?KW{TRyjUaPx;}smd6em3ye!fgdQ>96WugAnYJ%&aJ0l54R;FO?Jf;tHV zYv3oMKUgV2odn{20sQ#d05m12lYr|kq$xqA1a%UyjP#R$<$z1%ygj#Z*U<^w2JS>| z6L|u+iTrllX2=t`P2>sOCh`Ps6WN~Ie0}mdc#A#3p4nP-9^N%z(@j^rHqoN@CONSC z5pPVcO^ucVi2(G!HDQ&nNW4eIy9&Gy^uyN+Uul&uM7*uhUjVhTH4FDy-`S&cG%wke zc;Vq{iE8vX%3w1PEG5~+HRvJDE06YYHAf>yxkpLXtF`_#Tg~G` zb*vOZRyB?nBV`SicRdX~Bwg~#r=f?XOWyo6^oVrH3!unv#*=qtk>4u4Kj*z(bTv+#m)K{NOkQOPe($&M-Y(@n{;nv3@)VCFgI5g8dZa?iBvc zR1mw_#qW67ifYW?zV8V6_LIN$|0Vc}O841Q6z)=TweAo3M7cLA_@;s{p9DUs@bkVS ztWxo&{g>cqm9Fm;g{Rti>F>)c+gpndXQQX(?|XP^^5I8H{@+6>|5J?*KUebq9!mM2 zYJ3=?=y_#k)<-}<;hMe*R_)I0hG2c6XjcVM_4ve%D28rRK`@DFcK z)eGJkY*x>9me_6jfXtPTq}VEcq=%j@MDSZ0R5*lz-(Tq0t)MR98)>}2W^43}VgU+^ z9v@|de{QomY=sLRA2lEx$bb)s*oqe1I4U@E6fXF<2YsHtg!ixT<`q5UFD$@kz5Pz) z_9BGT8*BI#0p7)8j2yx5q28hwURN|+Tt1^c)qep4RrKmVKc6tWe~6(gG8;W30ByQC z(rKiQzVW6}re}mXvQRf~(G7d{suY`%eFn_n;dE`z#R%Ex5nG4!kY}=Pc<^P_TXfx+ znZuNB@R^>xw^5$M>+Uqmy<*usT zb-3qeu=cvoKGhZeqEprOIL>p&ak8t?Wz0F#jgTH=`I$b(Fa!30E56F5z1Zm1Ui=bI z3zXo^;-hgFuFP^9uVzv7xT%9>@ekOl%^8LFiarzhHD^L`cB}x;w3@XrgEI^9Wputk z6if@%>_q{PG1$0lKbAUdNymo7g=ZoaQX%}RBcZO>j&bQfOnYivKRq*{Rl~Ibt zD8aeJ+i?a_iQz!?f-lH@t!52?r{KUfq30Kn-cW7vz)QSUaqt%!JBB1e_WOT%G(Na? zUhA0uu|r^;k@gOM%`UCF5`nzIqp4aoZwf=eQUufu+*E|q?z_?N9!ybL{WW& z6kJu!wn_+gPugH(-h~vFZq&QN5$!6Y+CeFMjXFe~b9fvAS8b=LhnmYmyZGD;^H-{UQFnYEztU#eOY2i?gi{op4j8=>`4sJ+ zX&?7Es<(XxuxL$XL#ob)v3)e{mka+fC-pyH_`6TyKk_>4P^j9wP$&VKSWiLRgBHuQ zKl%F<=*$|~(B28};Goxe`H@!rPt<_mpeLl`4B<1Mj_LicTLOT}Vcasqn9r;lHS=KJ zWY*B}7d+{K4AQM*=r0a)NGrM3!pM@gcUc_p%!Njht;%?AYwqh z@bn4KdU|?<=g=h2E89H#!CJFTtLDk|@XoHT@Q$v=mJNq+Ynib%9JwI8=Ynwb{50HR zmVE%9xpAj88NL1uOWHcuhQp!rZrX4-ZCm(ID!jJ=Yh?eEW+v@!BXcjBherEnFl`Q$ zvDYr){+MoDgcapsbg*!Vt0FM5WJ+L?FXTrH?CIC&agKNS-Lpr!T}IJFzu_GtRj+0? z_+i$}h1szJdio9UT%^_PWH8Ub0iMgWnh%BDIlz;z)vSZP(L)iT3HB?Zo-wB9&JM|e z?*~1Xk@Jl;m5761CE`t_98_JHXbPVWqgTz$!-kN`@==YRLRt`2^k8Tkb<*=&)!P$0dgE>K_D&lY&IVB=B*fcn-&BFLj~EE}XS9{`gS zpwS8ti5Gy-W}Jg<^aRM24!ipR3Cr$2fVwafH6OAL-7+Gn(>HDzh3iDkr@y97yN^nS9tNk}Z$~Fz(XV!1FQ#dA^Hg5Z&G#8P zN_F#}qMP^Q;z#uKXy|)Aj}WpCH>-ss9u|E+I`f|FAKcpO&-8i$fm~I)*d2%~_4alW zZ&sks{~TMT(3sN<&K!Y0|7Flj!HeH}+KIIn*_ zJNr5NMQ6Xgv$pt4z2-pRZasJm{X7*EVeUp4J^Wd^Zfx=zP3){i@T%?X=flx7V@`qW z*S8EY=e3X$E^!onzGlq(86~D0TRA+~FZ%c$La2K>!H-=#{H`)z-OmmmUh1{fKRkJ=K!ah6**K$^r8ttHrmgBL-q0#53!d&hF<;(1Wxqw^B-Y( zjFJDjqYp8pu@=QZKaY>DC{RJ{yD*Ag|Z-DN2Gc)p@F)z0j7N1cqPPzMx zIk_2>pJ`8>!zs`T9X?G6Q4`myHTVPv>GJ>r;~VV*X~8d^y$VgE%UGcY2VG5^R{a~8 zpm9Dv=LRv~a0jLB%PsU6JhNR0XS_)u;@MxZRWf_@W7G67o>asiCXFk&}@APzAvIWBY3SBtj9dDsun#_0L5;_kVmgtL4mn$ zF2-asIn+%Ib&bzxP6`qE47-V=GoU4|${7RWUA3BGiG~IA?B#+j!p%1<67g zT?TG8oRgaaDCKQ@n-yV;h2l!sw`l8oovge(~%pFJ(4Xf&2>Mv}*Rs$jrlne0CGMT&0G%P$Y8= zoC!6YjHpn<(3{AJRKszKZ0rZ7eKhRA=IG=1fo-dcsD__H4PVWpBE@Ft<5yz#xk%up z4z5Io5<2)lpz1Or?R4;zk>^-FX+#g;979dSo#Jg5Cze_T6&`B>h6VVX0{Vs0OYA$ZW!?Ds3}9DP z=gGS9076=wMpAvio#7a>xO1I)2#og`89J{H*G;rFWV0ily_md1Eq8#iyiqqVG0kFq zlJR|z8R8eJ$bu5Cx$)&XFprIA|GonfsFrQ2%lOD!JbMMQ<~7oDlWE)x8he3PG;Ris zj?;e*dF&wlMQw50r4Q+Uik1W9WO z_|p$ zgRFUAzfRfzL)LC)u;a6K7LX!qAET#=y@fnJOUr9Y9?T!w+XvW_*}6UOPklA z+Wn1K*ukF6_mur&J=i+tj;?||qWE58yI_qM_g`U=2i%HhZ)9M}8~a1B$JT=Q#k7Qz z?#og2qaQ!U%S-0hA|n|Hy1il`EM%=@z!9dt(Q5 zoyp@R>c5xw+&4r2L{8CwBv&J#&Lgt*JdHcSJ3^(qtX(r?Q@o2U-+f8%i(XczxK z$&c%G|3Q8!q~DSJ64$kHP|6h#EI*+waN*n0%WQCB94cl?7^se-6Eqx14wJ&=x%oi2 zXD`pa3{dOg?&+YBrS(YfNlS|!+~cg+t@WtRbs|8i_Ca-SAxVU?ue+VoiHipU=$J&& z+h;tVtJ60)Xu>OiTJ^nP78;ft;EHGW&l6rh#;6E$sRHSh!}!p-eAf~?qQtZNA{553 ze$186Sw<8`togo>{qSUroJRhDE9I-YvKO6!QdE@sjD22X<*?xLz03mAd<0X+wusl5 z)xu-&D)(jiwn#SOqp?QAoOkh`ue3vI7^2|z`gpbjbkIL*)k6??m{EQ4LU?=QSXW<(%WaQcI%7SC zm+`<=;E3mC%F=$$J>Ug@+1(iz1AiePG5z`7OpG40-T2-~)d~t#jQd@;a0V24h|Dvy zznBdwp-dgw#eC^!AXaf2I7PS5uwMu_(0}D9l<1#H^dH~``V|X(jzm8#(a&4xG>Lv( zqMx>&>9+lfHv5v7sFuGX8ncZqz}yW?>@c5ou> zn~oEmSJCpo8t&VStShn=P!~Y3yA?I$)oPD$FuK)>5b!cu>B&~WYt6M##ylF z4j5dWMI}X(8qa>EH{&&O`Y$R|%_|6ixX~K{FWrcW8_Ikbw4sodLypFe)q9OOwneNc zfOV91u+C9=IH&wmbQZ3+SvepGJ9IgkgSio6cM+M~#&p_Mn4;fsrrgZ@)SXyo5V7e6 zmSq_tMsLO2D$&ibe~I4G3k^rx{9bPqU1(zbTTs`KxZh4U>sz$C6bwk~Tkc6|nCms` zTC_zygu7G=ryx=#N{6h~x0K$S(m?-G?WGY!E#-ZYzg4Yq0rA16ln(-_i&7fm zeOgh%c%RcD`*@#pVIB~eh%h$_a|O)k`0hBkwSGNDzY7W}wUmbS&g;-yI)E{WF}DDD zpgtwgby*Hlj;y6L%m#Fjj++ui0EN;v{tU=mb7zWef5qu~IGj~_FdUEXuSnhBpoiCF zMH{f%$Mi5?^qa1Jy1CYCZV=z3VjBy|V`2z@=XD+vi;!Dg!JVx>bB*hO4NDCh0%y2P zXNG@#Vfw+%zO)Vc_|?AN8@%HihM7IHbaP0Ssb}bES7#h>rD=h#+G1B)u(`#Tw%S*K zMAqCC<-Ph$+r*0AD9lhNV)imMM>p$hMm?=`bhff*=+! z*sowYLd^0k{+e527wYD^NZ)h0(g9`#kHjlFN3O?3f-!sKrvUW*JBgubdjEKaC-FPD z7t^txxjuZUE5naB)aXlde0`hz+TcwkKJ&aj?6LdaZMBH0yL`^kS)m6cDRn#cS6@?$_Xs9zGk{VLG}I$j!QeT;nJzl=&Qd zmo0>FBmZQOQ0Dd5u))nkY~9=}y2SOAZz%f~c<3hPv;Vq4xNjHkb-LLS&)zOkzp+rQ z@$7erB83BiDd?F4lgNz;da%y~qzIpPBU5{lPNpC3I0TUGn~t{2iNwtMVDpuISL>VJ z(+dvko7U+CTXkd5&k;^Hdi{bRl*tz^k*m6p5+sJAKwSOE>*5AXb5OQ|HQ4%+63@QI zfiQ;s3pC%zLB)z#x20G`u|?j+I^lY0ZJ?F?!}wyc=i+U3ic*TJ`(P`Wl+aWQ7*bBhhHxIr@-H|+nBLX z=3js^vt#tAGwiVdUZJIx=w^DKQHbjj^H!gk-elUF;5EcZ2S3kG(7*FI`Zo*ur!hYR zY!y9%-Quy!mQEnAxiuoTCpaJ;_##2w%nTsBOYE`H&7P}G`(&iyGSahZ+p8N0n9!D9rvoX(2kl%J$-w?P#Q&7>RRgHU zn=x(!CEd(?pLn*4AihSW^?*$4Qjyki^`;gfZs)x>y0+7t@{n-gzPug>XP=%@5+)w-Xb}JPjCgb3FSkIPs=_ z9yDAuO$jw$AH5AdVWM3`dpOmh8--{a^^6-_Sm#D1&-a>N`c1UN`*rgWUyY4wo;_-_ zcfcos3s?g`M+>8y@5MjxrX6FiU#R07PG}|aVjrz?=Q>iDfDSu zvw2gV*PI4s?zDJ-Hemq8tVJmkz3(5 zS4VoHj3AkQXd=ls0;nOmB;%9p#U4Kx`#z0k{7)a{*ilNAO(A52uA}+Wu2tl z(J`>dwpm+je+7|i*1Q2e*fe*;Z2Qx2N5&yjksl+N;zG~JdD77ZJnIE(Y)@_5Xik)Q zq;Bp7Q=R6P;Gq=l-aoPFjX@M(e&3pH+|!wvL(=^4uoB2`fIE*keoPo`)4aOat>}uw zQ!r@=u8y1avR`9&yeP%H2}3em4)1Bk6Vb8+Y}v16`CGIe_BS8GI@<|;Ju+Vd-U{=Q zu=mJZ3P|=Af;rAx8pPU8WFTJJfwj~n>{JUY6}D&p&(V0rUHW*t4hx!7y8zv@s`?9! zE?y!v+Lco=!q>N6d)?Jj4;AI;X-%+-3gioQU#nxB#dXshb69uZfN*(Nb1=OvTW}{HIp26Sb+GbMmxYuZ(y`3D{f&oE6_B2Ek z9m!g(=Ef_!vJ=~fx)jrNa}xx19sDCrKoqR0sl!`!^m1V^lJp=uJ1BvDf|9E}r3|mR zI{Ftx^qP2wo!J|{(b;gS`pYlL=?6ETTX%3XzUxP|T^~IN9as8}seJ$7o|HfyjWdIX zF|}wL`$wi=@`my6l>pjeS(lj#8#$1$ahJ?|<}xoTHu}msim|AeZOU;x#_&G#OQ<&| zrciq{>(L06cpN&q+Rd}^dcyg7N^sENd0=dys~o&_!i%*=`$a&p{zRTb(CWItt&tBf zK|&eNWi>h<1&eI|mzaX?0f=N0!TSoKPyfX92mx0CM4psQ!;d2uNiq)jKP&yOp#NR) zZ?wOS_u`ymJ1>nxo>5F~oz-!zhF`af+(4oj_6D)hDi_1$$R+SbhA=5Qmva0qhOB67 zmNCB9d^f%+ZeId&l96paeDb_QwP)zhL{sLh{nE5oBTm7#nz~u(1$D8)_-eym4osx~ zVdSN@6`aBro^zQ^i%p+kA6Miml*;Ip&X62CIBVaF^hNdxHactHjd(;xwfPO`o2`sE zsaEXs<=jj3@oI>ndU+?C)x!`QYLC`GdoSM9&Z692M)~l*E!diJ$4?@Q;V?JzjT4vH zLU4fv=7%VMd^hIO(Bhf-L$^`f&9ODr`;lT6*n?3MJ(RffJ`$nvanl`ll~ZvCwl|DfFZf_1%0oAr3jgiI!n? zEYTLB(cn_hljsqUKGu6+iD&D=xq2CPj<7C@!EHLn!5k5xf$g^x8~JPzqnqdnI*TQv zcYcg&@Zkecj;+v6u4wne#OW>Z?w$5v9cGXAYv?xZ&k-2*90X@bC+z0|)o9;8H^s(O z8|{3*$vW9+KcH;<4zW$Yt&a8WbKvyG^b}TvHoTd6)kG5Zr8_JnMxmMgEhHiiaw6kg zdPTYV2^nD=v6Pg#nNfDP>s2xHWHFyY&w{2pgD+Xk)uNnVAohq<;Enb(Val>)!6wVbO#rl6*=5;I2M(i%eDpdgaGF;lZ`W{16U|@4 zdcJSZ!OP98_OkdI5?7gYs!R?8oyb2~C?$-asZ3hJ2qOT5agNNSc|SAxYnU>Vx537& zVYywmP0!-mU!koc1Md)YU0LouM*{*=*f0;g8u5~gGX!oF2Qu!NV!eQfF`fgn zMwpD(A8^_ah~t`iJZ!E()7cdf`Bo4k==3_>NM9RU3wF%70_?aFl~M_ib|t_%cuCK04h_qNfWoJ1dS?9!=1s(b;z zZ=lR6zEc!kfL4ubja!m%hQP5O7u^zk-=MFF30?{sGDM4%BFKS;lt89fg5*%!9fQ~* zC(*$KRo@cai|r%On#kv@Bcbf^VHSpnGejB*|4UfwL)q&R{$HWSuMhV568diwGuMof zdg%FFCptI0PvgCF<`&rmJc*D{o^*5FJ3kKgsbd73K58l0=vu=2`y(;<69Ktn#H^bO zB0~S>Z-yt@RiS~s=>sAIEsY2@3tthaO)IeLIuxM?^@22Maj=ozgTaVy)~u!{%2%ix zHB1I$&yYa6mOR2#MMEpLy+=kcTPg;s+89*7LdSHz$PrZlUuil_Oxm*v%y>;-|o=h ziII00gsyn@J?JtHxN~n#S%OB@hVw_cV>C|~@fkiX#87B8SKT<8{-4C0 zbbtFf>CVKQv|@8`Nj1ovHvPTe5>`o@H7$hhqj}`|&tg!O!ZE@o#=zJI0Vg6lP5>8B!Y2u0s609 zW92YUd&+Hw8|@q5B+IM5ra+;h?Dm=hzn=I*P%TU+#XXcHOSvro2<#rm_sp3-8UWpjN)ftWUGFZ|O^?)bhFzl1y3Y-yK?P#=L z4=bc+GwdT^W8={e%@xMTVth)a6|4j-sOxWk<~F=cnqMc{1BWG6Jy?=s)8DxCsZTC` zv@aCNwEwdhqy{upl%a~|U>`7BP}74oBTrU5n|B+Vb^6a}e-%Ekhx`udUQR!pB?$Sm zu>ahl7rSyI0~%91~g!nh6dk0iD@GHx_%IBu#ck?tsoC-*awC1 zx`=fra^1wQ(KlwKM?M2BSn$<#jvEjrifN8)=i1is^-!-J*CEBw&$j`@&eY8TTyMv$ z@x4eUdReS!1RFB+0UzVFoHeeO14-~JF}INK1VHOglGc+a&eS-;FoI~oWy z(w`$Vgfw6+MS)&-{rY&HTBumO&m3XiC(OCRd{CJ4g!zy#9}(ssg!#BIpAhCkVLm0y zXN37DVg6Z|e--BQ!hAuPi-h^IFkcnsVqsF*<9(J1bGb0z5@ww+>xH>On5%@jTA18_ z8Sk@RnC}X+Nthdj`43@!AWZJtjQ80p%#Vcmu`rv3xkH${g!!2;C!)&4`-~Q*PnhQm zv!5`3BFyuId73bJk}BSZy*}O(3X}8Tc%KYmb`@rKVV)+;EMcB5%$~yJzSMZ19ARD# zjgI#jDD1xwc84&3ATXB+^LkbSn4@8?Uym24Cp>ahRH&aa+TOp#o1G(3-yFKR zr*bCtqnzn8{Znx;2n)+2F|za*MqZBMn4zqo>rV`OC&FUUcqB%J{xOkzItnX~Hv99@ zu;9V;$&WV%o_=cCx_hI?fmV%;Nvu<3I|e^ViJ5!QzZI;h;#a1yE*Cb#Xy+1{sKt?K zHXPC0_Msn3pd<4zzN$qc!tdII9}*`y6I~s~B$#_Ru$M3oRg^`rO0Z&&%s@U8Vz$2j z{ePPS|KjKqKYhD|zfkZz1v@F&tir8OaFK!;>cdTEDtL*4mn-N~aI}KIQgDWXa}<1B z!Iu?Wtzff)2NXQ*W0}vNDEJEn3l;pOg5?U{t>A+SE>!TZ3T{+zkAmr+Ncs*1^A)^8 z!D|$}UcspfRx0?Qf=?;9SivR*cPV&S!LEvr4h1h$P*-rAg3}dzP(e$pKKMxH_f-WW z+oZc%xyLAYse;`VwBk=x^1WNZc?vF6aG8P|6>L_pRl%-1lsptHRB)n#{%aM?Q+T(cdyaCaDtMp5PgQV+f|n}zX9d?P z_?d!Meh;X0E?4**1+x^~rTne%lU2IY6&$Bvp$h+k3V)x1&m0$j?&h>gj_{q^B(&&O z-fa{3y2nxlY*MhYNxCarB%G_@A_Z+vOF9k(oeGXo(4ud_{JQq(TKI1Z1LMYBetE^T zlJerQT&b;0~cH|B(nLddDE_Yn^vx5FV`?({R zD+@MTaq*-;pm;*r?PDiRDIQ;5GB!}+Itl)UHvF_e`J^eg7EcuVnTRH7b>w%0N@wa= zsJgP3mrSdWYAoXgCL;9-#nVax#Z#c)$VG>E34hCgp$d{(d@x{xFfAt#sbcs{ntqCiCQemP|xErD2I`l1wwBC_iH531x0l>rR(@CUt>tBR1STd3{ikpzl!N^M zn%@$^=63mz=_KSdX@Y8z70=*Kw5V3RTR|)^>2@@vjfBj<#I?!C(npJ$m2P`H1tjBJ__oU19uIv#AuQ{F&~=N49JVA9RD5m2CB|K> z4+;H--71$D_LP;DT6q%gTPK%|8#_5M9!XETG^964q`)wP@psrx3}v%To-_^SyCdYPt4 zDdCoD+dwZvl3uvzWRjUQBa@kNX415X8Ua1%QK%IZ)W4@xj-nz~K;#N?LGT5FkJwn{dQCuiDdTuh8J3j7+4@9=6z&gWLmdqGvgu5`(8J%QDWvn7 zw`|$o+T6Xp6TBoVb#Lji(8HE7Wdl}!G;Z~W!lDN^`r!i$Sjng*YE#lIWaK&|)mUz% zMtVKQWe_2R(T8=G2JxOm49_zCSQIkf@v^dJ-xzwlTmJhQfi1+mDO7pUFFg48$2ZxfueT|Ju{RMoyd^Dm} z{Q-Wn`q*6)uzL4f@QMTnV@Ymv7%1B5U=0Mr0ZTE+&6{J>GR2j&L#Rapb=V6mSAS%4 z8vnB%-y`$&`2LIF`vhO`-#UFjaF5_I!4EUEmknRuFmMKlqgERah&D<7Mhu# ziOVrhd|H<^Hoh*(9u>5uEzWqRe0pc1NwIo#(pG3v3p#WFEo}i~&CBB^3LHGNmu)bL zdPn1pjbT4_vIC7)CL60?O)%^d zX@R1+KIn@N!fnJ(5|&NV!cIv9Q^YbJDj6+&8 zk`^4R7g=jBYV_1M*40@%x?43bO}=@1S9fV?B$ABzlXYuX)iq==+ul`Lx;4}nk0zr1 zNoyCpO3~qjb*=H-u4UKl+WK`Dt$U4~XX_Swt`(ieYFSx+k9>az)H z28|fEQmigo?eVOxtFK$zuzIahp8M91l@qf+Thtxx6WR4cXQ{a_U=-4CWI#mwY7O$ zTWP2-Y^{kz!_hraYt2=A`_|T5YqqvoYqr-}Ypw|)DjG|!f`1gpC98siy}`gLYfby+ zM4z>0Qzt0;8XuPV^(b*0nUt7DkvmNVJJ12o|T^;|st@~2XK2L+E z&cb5~R72y^wbq(l>kzqXH>dEJ?%Lr6kFKsXuQuOS;LV%byFhNO=~%sb4gOEWQM2^> z8VvX~*!P5o0;r(G+i>01TmV<0t-WQ(rcxP(CbGG!tG%<^#AejIbge!XFtaBG9jUj> zmsaO$=UL7|=Sz-J$32dl9DR;9$Jvfk^d#L+Z^N3qnLM-rS$0IOk3@$fR&ZZm5F=OF zq`m%tS>shiCT@iy7WUnHL(!pxg+T}lPh1RUHDXC*VZtf}x6lx*-Mv9etz8(Neb_c* z*oLiy_98KtNHl`6V<@LtyjX5%n^kOvg|NehxayE@z@M;SiwuS$Dl$0`f`l<~#1akr zL&+8AS;L@WGe&Ly0z6;vTg+CgFUknk*Hm4^}K}+aWm^M3_%+5UmH% zSq``gKngOj?PEI(eQWLV$9ZNpTJD7EMR$TXn~z;gy6u#Scx7W&Yv%GO?4mSt^HW$wkw9k9?H1L$;&CcVV4 zJlFPOCp7Gj2f$$v>KNOGa45O|HPBf)Q)CFLBhEGN9qPM2$hAle`(x-gR3aoq5P$~^ zng*jtY9Ebrg?R%V->X!jZZIC$!9c=FRlyqc?^8)+xLqIIPaSXt>;bbR9MWtpC-$Iw z6G>{}LCb12HLlY2MBj2pmU6c?Cr~cWA4CWRH&Yl_V=XtDwV!Qayz&TW!wywh$}p2L z7u2*%4GwN3JXV2A?Hg8HtL>`|f_nttEx09kOz;hYGL<7m8<{gZr# zofW@{-Jd!U<#q*=-O3bE(art5vq{1>K)#&uDfX6{Flx-X;V4_hkENK9JJ6{60tqmt z&cR(T{sO`WP2bqa##2i9sAG2Q?*fgDS#hgZuZFG)1u(cn{UOYTRJv*|^15_;4M_b5 z1V147ErRb8{0_lK1o!#+LgDM<$%Jp%kIkk}d3wNFs~EOJeHfX3I16ID0-{BD@7c@! z*NekRK2xNjNOlZSYOPpg2IHrb1w{E=Anv`x`W_i-Zrrsvgu|bnd3Zo#?#emSJEmJHrUKWc%tvz#xIjU$27A6kL6RUHXPFk7SyPS=O-e5428ul0i`D73K<9i^Z z9xyB1L+9CSfdmFsf>l(`a&;?X{oT678Vv0jNQ$g1_2l+cO=k@csHJ0wn>S?+OvxSN zR(y54)c*2%?CN>fsdt}2_$j!%18iDc(tR0f8O1uSE+)JeKA4*w5d>0S9WZ`A@bd+K znC_%U9q&LO^`*nd>7BIKaT9{3?~P-<=athJ!MO%e?@&J`okn0YU{Vc1ld2?29o7oY z`)QSCt$<)dK@4T3v25{rd6xuRJeF;2T%ov49zFP^VAOSRD2#=}oDS;(qms-@Ssmqw zV3-$Cn5g@GN(vomd!+=o8_YYyU0YI`E`c4Hsk`RMhM-RCjRw@Zu_x}2!JSD`ZLBe| zp`?%ZO}yTicb`(5Hhe9Q9UESkg|ExPl|R!)j~__S$iF@dzc35mkcD5wI2Iivp#5D5 zYfJO7uy&k}wuW zYl#_@etiz91}J++`og;J*i7+<_wcwF7zA&|aT5p%7Z+@xq7aoC7l>x!FgByt2Q%Z; zC6%l=IBwagl))d?8-|98>%(>-3IRC1KO7y#K1B_t40^~T*D0ufPm=Gs;EaWjcroUN z27Q=B%Jzqm!`H=>8DfEdrmeyp`SzK%RbVb~bqB3WEICi(*$mHqW21`4jQ|{5`jrEB zri9uDiMlAi@M`K;ryRqOBpt^m9($AyV_(?-ytQgg_KD|+x}(uBR&LyZ(ru(~iv>vT zhsdg?SgL=Zrqlxkj$3ZnDM7`icd$5^VP9Bdqt-LLvD3SU93-eLZdTn8v|Cle9ANxh zZI4!cN&D%v297`vhiM(v(Q0VmF#a=7N6N-(H9M=6zmsXE@_BOlRQ^p)H_ri|-cNXX zy#ec zz2QP?-MW1z^V+&~H*MVA)x5JE?bg-KLD2vz9oZ!>0QIb>6a)pbw;&67w|iT+bpy6; z*}7xPRc%)xuDQL7Hm+UY%KV#m?xxPR&TVaY-?+7zTDEr3mTOcN+Kl6-&hD*Dqsjzq zSX-x9khgnRXGeECUAe7wTZiU@w2jo<#{9b4+rd!NcYtk!%D-{PmMvFX9ox2dwzYR! zIM8Z^D+;GFny$T5)?!rR=16=kHp~!t$E8P8x8m=**9%e)uf%=czAZ&8r1LudurXXtA*1$+q@kt zct=Y&Wbf)`L$qttZratdZF|SIovKdW*6rIic5H%AI(>rI*{c0%%)|buxjtWyLw#JmXShZ1ut2k z%folqivJ?P=L_x;{0zZ+1YaQdsNl7Nj|+}ph*bQVYBhiSo}|KKf-e&MKEcltd{l7! z(4L~Fg*yK^g7*k+34Vv*iv=GQe2L)41YZiA+jT$sE%*k(M+Cn}a9XVMHwtbE zezBn!{1U;(1;_6+sPdbZ==?l?uu@_NhBZ7EkH-ACM9S_r?C`N+Tmdf}J7aKjV2Gp7 zHlw!gy)Nx;o1p-e)bolmV4?>4<=i2p}t!Ze*H%71Pgn z2M%LE+?dYCubfvU1b~_R{Hl3yQkq8#Jfm=dW4*VGIYJw8`!8&@b0r$BR``+GZ>!x7 z*vRuPakIJwu_NK%6J$pM64@YE-bFEh`JKy4(}-JG*(>%8*RJB5DZ$sMw!|(-eX7tO zj}F4!xi4v4mTV(o^Y4%E=knNM4#V6FU>~HM&6?|_mS>-PpM@4wb%p45kF%w3$1$gY zC>qB2saGhcw$7{edL3LFRj{fY`?rQ-To@iWDzSfrrOy8NOReSZ#B$=g>IfX|NF0Vk zzb_Dq^J$NXYuk@pa8Yp@@22$~9eAd>vMMH{YMK4e%?Am{30r$Gb$rl^qcnZ+KE&RB z)i(CwI4>dmDms8{UFcQ%%>YY$&Tw`I7m{t5*P=nQ&&^xP-Im%ZpK=z13@@* z_-X|%yfi8=O3LHTEx=BJUfwEO>P~=h4C<(~ze+WrO#~TXI1uFhxy4&x93X1B8p-;seyP3_hR4{=wd87*75OzJsAHr>VFRBT4w5QeLNY1ybj)>`c=4mWVP#S z0;vPfI*(_aI$8=2v#$pJQVcx!jkujAcSxGGdh23e;4XjodT!s8-<1!V;J)PxG5aS~ zmDL}V=aX3=rc@?9sj!j+dn|3nn z*aDpe*s=pw72rnoQ-Jy$kG4g%2IMlXjt=SLB{&7IS1w~Xn|g;L!`KA#OhiLiYW^#K zp>dAD(F=!v>Ik_NCt54Ix1MLU4Mj6XDtMS99Ly(0Z%^00N$ktQLEdMfu3bsElW{h; z$Li|X+Ge%yO9msXa1z!!YlX7OamE9U&X>DblczUzT$!vVC7&0}LqmgnR$gymEKV;} zD(jR2rKzxMI1Rx>&Qx}rd|RiJ&sN7=`KPTPhNSvf3a5S8T;VvLZz8A(%gcm?OO=?! ztg52f{k19)3-;k)r7ts=O`xpTlzcYjrMkQeqic_~4dr*F`_jHc*NpP#$EIv`ooXq) z*RYR?G@rcr)g~@?o{Y71dYwL{3itB2TtN$J6PMTd#21ISQF!Bdg_a{{p7u)ntIBXf zs;$h}*4@Bk7yF?Mo%Ao@ztZ9jtP%#hSKR2xXr+wOv5)xY*RO7+g`MF_OKp``TG%PC zw6IlH0%6-Zl{8s{4`>T7BW#i6UMSV6LB={ceIB+ajSZBWt4)l5cKawpE~WbtS-oK^ zSYB1DO^!~*C|tfUl4xnax{C+c zALj2ym-e%#8Z-(ja=Z%@&MWVnvb29Vm5_Qz!ct!%ViMBt!Yg4?kYC;vuLL`*A*Y6l zcxgiTGta$gAyV%u-o|gQepmc_dlP+1w9lbs59b zTA=aicNLF&(ENc9+C+WSv$P*y?Wpzw97ef}_i;Mo+y~gFsB}P;4ye)@=XBho3p2gi zkOIbox|cXkzZ-c>%2M1>!M+c20Y;*L`0Ov9On1B<2e=*)!e?F4L0lH~*_xQ?v(nOy zFexqFyp!{#n=wb4?(-oC){7O;nC63ty@=6R*VSF!?OQvyZ-)={I=uw0tGf=uXFbjL z9>LC&h<$N~5T0A^qUUN|c!R%D(q`eEeNi{AaM6tmGt<~7)(ywq0i+#}G(@TA4)gL) zhSr7JU!KkO(2Ls7eV@SWbmjcv$f_dczbg^ALHzL>j_dq7pC?7m_~R{wRD!m_nSxHv z@vj4M7HG`lvAuH~^DqzY;NVx-kYkdtD`5HVvpJ$y9S8e?bp5BJ_YuEdGH}tON`@exiKwkj9 zZtZH%b-efmuFHrI_4TdY;9Fns zTZ}JePq;wb*N+|K5#a1&UXA;&%XYUmri>E%(zyj zTag*3@_I7jwy3-qyEa+2D7vPMIIp7HoEfL+ycu!bif(sioTBSN-0l|CH_mN>x?(P{ zk5%K#plQ^Z8~r667w0(EDYG!NgzknLZV2@92K>YYE-iCB*|)5tS(Q7H5qFI$_mn-Z zThq}>Ti$L>SAw_=11M*omapj!7frA%3SWj!Mc?1)q5#(+sA=x_kxS`4>1TGyQayfi zmGzy(t>d`Oh_j7{dXD4YOKQWRg3z%$7S3xIJ>H2p&0fuUH69pWhWj6_h4eVQY^!?{ zyj&8+3Mq+Gqt*P$D%zAr3+V(nsr>9P#tjel!>-=b9||OvY4fkDNcmw`C&P+|%Z4ul zE%chwU7Wt`8r?Qz;j{#U-q(qH``pq^MbyK5u42C z;BcgV4)iDba4}JX9e0#qZ0npt! zfSf)i_|t;_5s>v`?UF+Gl4LoR6%;-Qsfqa>`ihdSuG6!CT!)_vzL^h)@Pq+5 zy{uNZ#Z`dla&ovFV`G-v*?0AIW13?(bA(OvxNGY?kMF zrMb~|Sdkd_0dhSYJZ@S{X~0i&+)bss?|NM6z?AN`F2p^?4Or*bO8?pLjK0aCyO~#Z z|8N}h2ybB?RFIjHA~W6dmDbFzq*qaV&WRzh>3*LWWDEK~hy-e)-PYgZJy zS0pq0^XYG*{mNYKGFd|=nu#uJ#kn;8uJK~%#0*^4kc_x&8sMD&O$e9d;(_!Lq`is| zK)QV_u`Jm*m&I+_hV_8kavLDyp8@3O$Ab5_=|1@^U;*MCx}7IfJ9GPB{ZMV&DQ#N; z-FVD}IRahPX=_t?N8Rb>P59fQ2%dVEUW*Qc@7!XVn?&9~DHYb^)S2UWGyrm*)tj-m zLKr}v#awQ8aYme-b{upK9Yl99f2Qrk8h2HV`>NzDYCk@kE<099b@erL?$HY9!a10W zRZ4%&m33j@39~M!7>Zq?+os}52!dzRR*k=l?FBrC0lDpJFPQDV0A(x(Ukl@E(Q#fY z%BT|aANIwQX0DHd|KizB_w4#28bjVMUWM=88}zua<4ShqGLX}_oDR_QGYrW1y@DSR zybiW0T8j#fkFO4xFpS{(yc>5ucf%eMnKSU2&|s``{%<18TCeME=b`n9ZhMaBIxOFzs!Sa_ zHC}smN&DGH7ttBV&Qfw((s%n<&T?maIo71HtBL;O8tj{O8SU9Prxjd_yFoe9l?_^7 z3L8{$R(2ut*0Sk1=Et%f6`#xR0U3YWYjyj-4CsW|#XgNM1LSnIPIMMe7|rb{EP-u+ zb)*wpnNc%miZbyT(C|Fr{l&)-0vBTJV_%WMQ_lsDqq>NylhBs|HAkWIAY~_XAM4G- zqBk#`O&4~%peGBVCySsbF?O(i6{1ZrCs|j5KQw~8bJ-sZ;19_KVdJW@r%0YH zM6Pm>&N4K=4tgy|x*liRF15e2tUQLcBA)|oRDHe1zXu;EvI|klR$&LN$KU_zY-x9DFB^d4*l;iHlNWFwKM4kAu>u zi)O(dDyB1z7SZfvDHR7wcpX-Ks`}Twt2G7oHt0+`mvbPAKPEB+-)AG21KABd+XaVV ze@~#!=akbqSm)1xtv(y;d@9hZ__n=}7$=3imtaZ&6B{Pg=9p#z0SHqL23M|e0#y4ufheaZ@{EXnml z8{6uq_9KWV6K~pNru1x^bvnmt%L?3BA3M$ZSoOU^ni$dJw6?YkdsDW(QQwo;7hz3W z30*&dx^4j7wOrroB6l^~vkW$e3-;FNTZoR`Of+(dp5Ga`tWOznn2Xss=U?K^M7JS~ZYuSbOcaf~#(0;= zeQlxcwda<*&&9YqCs0ghVr;W5i?%#y+Oh|2WY+CNw-Z&oo9K{~m5p=Sn-PxYNM~B5 zH(Z!$Ef^C9@>s4D8-;C*Y~0WpXxpQD4o?T8hZrVogk90ejcFKl^*2LTM*xzTG zo{P@J8EEHge>ds*pq}mPlO`d!(SAXY<}m0=I@@_>znk%&&C}jjhKBQ+cO#pDhCUz6HIK9V zB3rf=r=A}1sxoq11HIZ?CW7^Hsm*L-I=NUex{uM=3 zaT0bS>XXr4?5}O%T^!aklgE^;jW7BEWC7j=sBEnE8h1O&Z$|l-qWp_c zegn#03mN8Qm49N2@<)Z=Z+@hTqnn=A{ND%2kLwwYw?C`Pe?6cozqQ8QigGST`4^-7 z3sL?$l)r|{hg@v;n*HqNvHbO?=QO{Ip4a?;0Lc7*DY$oB<0CI9ap-w%Pi%R=L|chVuj1;R|4gS3^eonc00T6Zh?dVhjEq@W6{&maWJ2Hz;TRT;py) z%s8reN#pYX`FTa~;+OR|@g0DWnmPr)8j$PuT0ke_trHr*3J^OvdJiDS-!1ru1pf$N zG2%}Ea=E_~{1hOMnZ{q>A3Uvqj9&rBq#s%Gl@F%=eh^dsX=V6_Dk7>{qHddI^xr zTmFj1KM2V9VL+yPUhuz2y6Vq^urrodVlKL{k1mvbG;|jB8ij#U-gA58{L522uh>2B zWI0vV&&u$3-UYcA*Oa(xj=A7N;=Lr!Rk2_3{043ITlhY&5I>9!S7krx@G=d{cr745 z%RtZg?*aK)JE8IE#;=?Mj48UqsPFO$S`IyR&N0*peubJO+8X``@o%9$&>~F7I;}Ic z^BQ$jqhFx+3j^qJtO@LYqA?m3Ut0>VZK!cK)X&2i|9qVNRN?H0{Zln`=E=D@C)ek! z3%Cby)-!?j(K<}UD)DcxV61?0B4{m**59RprDjEmm z^xEbccXR!GYCKv)>yOQ$I-J`thre_IbWgSD8YA28Cub(x=wCJOTmM&=HUDo~hSvb{ z(+-JvPP|=7hWYBiUmeABF(5y?1aEh0T(w6Fc4wpDqt_4pZoh|+T0_)%kG6qLoL-Z# zAH~|L`^uBrvzm2(LTq!Bhi#WmiDRErH}YaVLqEsJoDG@x!XlhOs4*}(ZFULGt}FMJ zO_Yomjj3H=PwGsH_jH9aep!ar*>STMLD$PEw#1|!FP1`Db{%gvZUqbUyMkQpMQ90b^b!c-9P5*nrWJlo%ScCim%pCNl3HLTO%6AM7zS4BrNRC%)Fl&n1|S$RI( z1ewgd$cA?N^^8(F<2Zb)XqV!nX!kreY}pbA-Hu>+*;kA4ZnJMQ()3vMI?gGhb56o< zYv&yo8ys*k#)5mPgT92YPw_oFONQ$+QE*PS}mCmru@czg5R`A*=m)-o$0zm^Re? zrPl^de^Anw;ZDpxCoLMlSbPwAnDsmJdk}M0=|bEUf(=%995xQpm7ZkCGIp0^T`1G> z?7!$$=P2WjnAGW@;eqjnFaHQTEuY~P6n)j{LHwLD_Z;w_1^zC!LksAuSWCSaS#I-=a@=oecFK+wQ2^%-I19wu^rTsv#?c0gtunWTaas#qQgaMgoC*Fk?Yr#3foI>}UthzS+0ClaK<*q{=SEG*SqmE0_*5#kof8= zbi4(~@u{`4)JdhUIww-|LF=EGqX*;nA9>efSC1ba!)7e!gY!yg-XrUA{um`WiRf|Q z4{eY928(q`vosy(%&@j~+CeYOBLYD|S+`zJni`FS3Y-&DVU z)#+7s2d~kvNh*%x{@k$=!fNP+RRcQMH|Bjw1uX&1LeSKJrVKi|5Nk&&-A=o(oECz% z2DD|MEo6Nze0ZFm_eKZ34dIDv#h$mNy&v&4Z^Ak_P1;m^YHJGNFUs5(Ha#Qz%Y(9x z0*<{|^fI7&Ms(X)fUK7lpOcw<4&~qz6Z$&=`8gzfMkPL_-;M8JOzcR$%XE11T_(|G zG367(_^Lb!onhN6)3fQla@8$b){6o8xk~UsI&5&3@)d znIFq4X&L%&5oqRuW;Xm=Wo&DjbRO&G{j6DZ7HH>zwgR+;pfzcWA=A9KIp`*YM_ijyDnjhFEQ?dy)<#F^JJ{Z$ zF@5^3{0sG}?K{mJ+>N$B@Gco2%HNWiK7#bB+jP3R`@-k(69wbEhSe^wa4!#(l9lAU z+_;kl`=b(b!|uPh5BW!LxA6xJ(o^M+aryyUI&4aN`YEK-yQO^Ai-%>7pH)t0!S~L0 zVJeQveW=5>d2ZhSo;=!b5OXYz>X05a7>pV(=q;KzM=!yOL#9m3om=}YA0_#eE-L0b_&;maRCDgXaNb!fcHL7zr= zI;#$MzRy7wh<|vBc&7Ang>M&o5^c^bEH$4KqDSETkGF_c> zRK>$qnem-2Q{GJX=5`-I8T8Gu^nRXw4C}azVpu)Neq?Pr*lJ&YM0nb!Hz!5_|7@v=6rd{?ib80`aN)cP9Qc`{4Qe9Q2xx zIcOBJ+WX*i+bkuQk>73`mSbPnTkI{GaA_N`40Z+lg|>5VK1V_Ms86pRAB^)DIex!` zx<2Wk-2s;G^wO1mdKzc#Pk##jzay|~R9R_1rLsfyS_NA+wQd<--DvGav)vcresSHg zQo7)H8O~G6=?sjOD)=Zcj-E7Y&*q=Vx{nw&4gi}T*8TnsKz;^3qwzZd`T5FcwM}!y z=QRE#Adj103a;eoly%g!y|$*8KUPMI@omAvK?^)TO*_-8S z;>Y-xFNE|MC`%YuiJ$*G-Ux+spW97@t*2 zv&`6M`@0@C8OQV3&cMMxS3K|QddrA^;e%$&~9uxXII0HQ|==Odbke{<&)O~+DAV1#*cp(jvI z*I}GD;QN`i^{|a_mcIyXSPdJ=KIR9IFW;EA$2YyC<@pXEOSJ!l_Dh`rWPH&tbb1${ zN(bcUeG+fKTU)!e#=RA~X(QxoIa*1V9j~HwC#z{eeK}Ph(|2pkccWt|`SiD-O~2CR zybQ?C4_?vu!ry4TZdk_(6<4oSS2h z3R@==UyO2|JLRCJKRami>saS!rmOE|z-u}lcmKD8UO>1b2M;bIBi)Q=qc@?;ME5DZ zW^|v@53u_ywM`#4<95XhGL{f?1U3hc#~0c5!1Fi3)fmhgMuR-6hz?jhAeN|E| z#=$&{i)xGu9N~KL{96Q?7yE2BK)#vqXIZ&zK3@Ub8*T9?K=cS*UJ3mM_%tB2EPZ#j z#;>T+^v?qF^Z7X%e=q9KyCq^5YX6cM=;euCJx~z(QJ;oKNQ-Jp*@m7GR#>w=Qsxhv$hUcX$pk zKQ-o5Oy2hq;eEw9x-D)6$j`3@?_Q?q_5$*Av*4cw zWPTrV>v#wIr&@COr_#P$y(erzTZs;owmN8Z+bPplh&ixI?q6~yB+2IxwJcYsrCnKTBCV90LbH|!lQXxfLxcIfE?c^xb~^xY!Nnuo%azN z?*l^n7$85dn7n6|yU&8XH6Lpb`!*}Fj@WsWyo7xoer+CJ0cvnO9a)vl<(x8=pS8(4z7f$-&FUr~+y1!R8E z^iR{v(x!+}`~gQU)N=h9ke{vqY&)%TOr&=9#f|3e6~;*)t8yN~&D+wQ(t*WnMB zYW$qbG`<#)(;a%h!e2&>%HB+Uc_V!!XZG+-VJv2x!z*7SzTZQ9%5Q#9OTt|gvmbLQ>Gu{7}KC-+5CX~RJUn-2_Qea1h=2v)Gn)VFH4r< zJM0py$FP%*r^-N&A`{jg?}>C@j|q=YwQHW|Y}94!0^}zsc*fjZSK(d20 zHkpNWpjWB>^hxv+_6-%71Ipk1AgMbWi?OEEb{Em#wm9k4txmcH^OpO| zjw_#rp+_3i?b_y~L(|aK&MR`yt5?74kg|v97nlbazdc?jy$|6pn#VtcXaDvJm!a1Z zZFA*H&lP9S!}9U?I}FIr+pf~<&8$w1cLDO-&45gIRN~L>()hK2{QMS>^~L{6{Kefm z-Vdnq1F|hrutVot49NIvc4|7M*K&O8t@Pv1(9bQ^)Pi}|2t8Z}eSALlOiOULd|`4H z#)v*6CD3u7j$P}d#@9HhXD{ywAk`G{wd-r#>*0f_gKj(jXaz0Befo2t)AYS+{T*q) z*qZ~tETjt{w9;<_1$3mxNh2^oA3-_X*BLnLw`pi}KQMPvytsW!{JK5n?9uJ?J3xLO z8qoN60QqSSX*^{khW6^fl6w`zK(zEX!)^n=y2T4$NFH`-J__V(W&03 z=6@$3KVOY&{5OF7d_SRay+@dXJwgrk2-VO@RkBB@gzcl}gtBy?lNS6+&k0I8DT;7v zShw3h6W8Nim8kR}mua`Ra;4{rv*%&?xNmL%uklqkX!;>Q ze$IS@o*U-@GM{$=^79ZNzx_hu8*bG2{5NVk4d5txvOYPP_AFC%rO*bfwGVa$Z_nhI>=E191#~N9?mwvUj=OdG z{eb+83jTG$_kTdAr|d`LmoSifvjuu{P8rQ{%%N<{e!8)<;nz9F&VTAMKMKfW<d+n=Ahqd)bG;N9;?FokKd({-x^!Stq7&&IVLQ6(zf!)59*pyP6-=ZdrEVR?Dn?1aqx_&=p}MAt(a9|PpK zdmh#}eMZxF0`hYTko%_SvpW7wfc)GG$ZyX`yytTouYN?+T>!{WJs|6#hb8_8f>(cD zr`IkhaxcJq;XSVT7KDDQ^}Gi(??pKEMm;zF5nQ*E{_chQlgrTUq~mg>=ZdrEVR^W1 z?~%H^{R_GcY97^fI{?UUqkv4e;EOsw3CQ^4f(O5(+c*iR;sN>j4IsZ&eOcpY19Dq_ z2$1O>75vnwPEXBc+3VFVu5>T1ucEVHpVUGx&BL6j#@ykyHsAQst7z-_F86%cz2(rC zN4^4meAG$ZZ`N&|jdNQ3>rT4y??~5eH~F__xD2k(vtQNi(E2rv{~TrU<9STuA40kO z{P`OizvOX^UjxYbYq!<7w_#nm0_)28u*c5DIlvOs>rB}5b79Zh>)nHT*HR_!l49?L zbV53To{$fXjP`-w7~_QLFwfoIZ#wDvZ#$`NM7ImS|Kthi0pRv}F&)eDDL{VyB=|be z^Ybp`;dIW+&ndyx-bv1e)bBW8-vXah`nw(T&ij=*eAA0lq>S^hhAMmqR*Y}Jc>UqK zIM`nY@LMK053|?%5w1_U(_MbDf)0&2>8}V)Z_#ziz$=~Z$}AdtJ(-s4@oPYSI=-** z4?L;yr=HUIvSS**5RmD`=hMk+8n6Fz;u?+Krr`Zlza_sQS^SX*0) z-)P}`Dfq23tOa)8#0KzL%=fmB;oSTv&dq=5q-PM8-K6Vg$5Xxwq_a$C|47T&1*l|n zSGe8DVmj+6_QxnoeQSvEVK46h$}@aMCEt#xHJ_IOnNP)!HGT;ojmV9Y%SzBj9SW#aRl@Hb`B=-=;hBLeo|!Z=;dx}-Nxyu-N%srS3|x5H<227RCdqkUN=_Gf zUv$#3 z;%(5=Y~LF?Q$M!pZ_2iAQE#4aozO*`p3ASm_e@%B_e^TwN8|QvQF5g3nAmaF-A~H? zWL+h#>n^4X;Ez}ef5e&4*Yn|zz+FhaPvLJnpUnK_y2Y*7t8~NX=_sJE6T0)?biZcf zoOZpdfR-U$>fu#7^(1uaaXnr|r+OXP_0YOW-_zD>P0Wd}np9ukgm!V4lN&NPCVm31?1;ff@hpppI7ca?-=eOVmzIBvXbUv?Z;aAs#%+EHF9D- z9Vvs12+3Pjo4Ye`mh+!LW9q`ZS@zB4TK2mE8CQLI5My$hyQ`|N>uleWXZ5l2E5lZh z?|%=P`i`T%Qij(%TbDTu$j=jkuc*@aN#qHLW5XY9yoB}xlK0nw%i5>E z13W16Le@i(!=t|=#h5*ab7l2Ssl2mIz;QgM0}@w)bHZ0KR!`@fRC{@Kj-J#wXZ^tA zU8kcQygM;#jBacJ&OQd%Bek%zYB85MIf&c!5DqG9$tzJ+_ z3wq~{*TkxOsvy6i{{m=VMu=q5m-6>J*jF!r-*3Uexf3CGMwjuHoK zNwi-z?#B1p+OgDuyIa8d%nJLVvUm}`*;L=S#0q*azOvg_(cuiW4s>Sh8(1uU7}K`% z3ei@dr+JP?@O_u9Of~QDeLr+d4Stuvq52H_F8n@OstvNn^Q8LloZ&jL>^!Qcf@Qd0 z&{;I1^uh`!+hHTJu8laLLveR7R@ehRrMRb1SBUS0P>+d1+|^UKi6_Tg{7!vmm3Q`d z#aQ`h>5<|?g)vu8fyY_pu-Jy+KT6buaN})aNBsl1qY}DuV%B)sSji|$=P1G{-~`sr zYL~mZU>1IZrO<1>Uqjh$`cMHKK%m?8y7T{Oe2o+Li7Ma#$!wEHfWM5e>;T&6440el z!_LV;*Z3a=bQQv8rYmu|OR$G@VSks+F9TOL$TVeU_tP0=?lZu94tUQ#R-(V#%^^=l zIpDGUo&vh;Z~;BR@@3(*bBo+_3uZg&31^ocJw|ZlU(xk^xPU&5V7*)C`~UHOVSCJX zy63~D&2HnxA1$B@5$-#PKA2VHo&~>432b_{3$tmb#21#i`401Z2JV&-_q~{LUgINz&2ik}D zLs@B6h#!@|ZN~O2XIp^KBfJ-UxtFgXaj^EkeNOb6X_#JViTf+`o@f3a^`CG{QZKMJAVEwnSUX8$?#lNBE zK|XI23XOqR!a*K{o;bcqem&^w@SO+pRrww2@4I+m^B=hkdEmdYZUm289MlvA%>kri z{hI*o*oPf-=o83)8^Rg*`$$RfS&O(T(BA{zD5nW=BPeGKs9C_wIjN$lyv}nM-j&VhV#V$ zDW4>*a#B?U@jr5s2Xc)g-vQ(s=X?v0?*hm&uYh6};@sfpMjap%-|-yByZ0}6zXENJ zJ{^IKRruX-4}PDV?~EN;16j}~9+Y9h{^7r8)KiK6Uy6MAMuz`})dA!`)CE|9`YuL4 z<8Pz%G=Lvuuuygkbs9StZ8IP34;jYLMk8piu|J|+HleH#0>;uPWEmfX%y{>*jA;82 zc0M`bPd*nkcppXmCYUdH9s<86w1o$741AkV#u)md=WM($L*BC?*9|E53FO7N;CppF z6DSkog?*mvD`dYU`@h(a#eTjR(#J3s*+19x3*4pffKN5cj{NUJnf0g}#>b)C(cXtp zx4r16uOQE@!14Qu{Jpameb-Y0i1$&9BmQkW{w=rzec%H*`1{HOJ&4D=7{!>4ao=H# zHi7RL!YGIBsOJvQgV#hcc+CS(^ySDH#trxBEYNR5*(?X(L;yJQ#DWM{;vIiium^$v zzTjqr4G6mt9!H2F96&gP@I2=E?Ei34E5cm}zen&Ma#0B3od{n*_%(w29vAf>ya(aq z2wy{Z3E?jYD-OHp3WONKM-g5`Sny#NZ9;e}!q*Yz-Rq*A2p>Xt4&l82bkQz^wH3&B%{1?K{5!@eh(Nzc^K=={D z!jHSC3*iF@M-l#pu>2D)@*%tx;UR<<5o+#7K7_X;d>f(UlP+pT*oW{8!rTX3v<2Y- zgvw93==BH>A)G{LIO3xH2uBgBKaKh$d>3KmgD(0c!mki&9s)mv1j2&|FCwge*hRM? zd>-Kygu2hT=-&{2ig3nfUGy4++Yr8i@Cw3tpTpnRM7Rgxw+L4};-WVp96^}%Uyu*s zg9v{>xcKv^3&Kwk++T3f8xVes(DtZ{-hpre!TKWN5&jF|zY*HLM_u$^5LfQAg z58+yb_aS^Ap=J!@58+J+pGJ5ZVea=a{t)&c96-1q;fDy7Pr9fc;q?fgK=={DoTpsW zjj$i#2*NKB&OL^_2>%D+X@s+Wfcb`S8^S{f&mqtcF(wdpAnZlB1L5NcqXK3za7X%(%eHTVscwNyvzsGb^VJzYo} z=pt&wAGo@NE~U%pa%!SxYN1wYqjuT|#lD$3=nA@$w$N7EM%&@Uz6#svF6yQow3Bw> z?~q`UvAA?3*AIF(=GH?dK>*4-AZq#chJAnJLz3?8{JL^=-qJdy@&3g zJL$djKDvwUruWka=!5is=tJ}$bcpVu!}MXgm;RGJLLa63=wtM8`UKripQH!qQ*?wr zjnB#+qKD}-^jZ2G&d~mgK2KkuN9l|7CHgXr(pTuK^ffw4kI~oZ8}vARlfFgY#`5zW z`YwGBHuCr3ihqiZ(GTc{^dowjeoW8Mv-sKDpV0I4QyQli=x6jI9jBkuOY}0GpkL50 z=~wg${hEG5zr`oWzr#@j|2f+L{%rK+mwWKX)BA?-H_0PO-|L5hLqQ+^wdE%B_tl#~ z+0@k0-rU^eyRzw;ycv9Z*PT|L5dM6-KN*eZEv>n|WoK^D*48!U5hR0Y9*OeuW%_?D zYtw+SP%Ib@MS{9*uj01d%`LmO&mT(;#e)G~a9>|AmJCHBnq-HKWCp#pbsF-?Ei+Y) zj7H7X$0B>SU>W%)7paESq^x^`eP&QhnWaAz#sHesI+-nNYqq36alLO(JUSG^-!qT! zpGnSceBXtcnb)^8(dFAVcKMpa_|we(WcHM}y4l;_yxVuBhXIdArrj71PNv&!r`Qxs zZb8>~g>INk=)02J!{7J%p-?>7l<32saS!-V`1MUZG4WTZeS<>~JuUg}=Jw{Db?el0 zm?~R*IA2cRHWuCAmkjL<`u6(6L&2s!;b^ZvtePnv4eJi?_xA-a(&e@``>x!qG6e!Y z>4waf;{mNClT-{38r7E<=$aKg;xw2ojG*lR5Ku z&7>MC_i!jVVCG3qR#Uc#uP+=;K#fiA4Krud@GB$FnOeIxe^`*f*rtk0}(NZyQf7S#*5;F<&B>!}1JXE&zJXw1h{tRw z5|YKFt9i2)FOiH#_vfu7FH5Qs13^uwG>fvcQmRu75j#berKAir*2@jL>ASSfRlMre ziVz4U{e1(RDxyywa;f$XM291L;{HGo4dUZ!VsXv208L&=^dN$&4^Pve+!v}%`+1Q} z_2B8KVYRBL8tOGKlatos`84n}d1s=0r)DvQ-km|l8L8Y{e5TU2so}pTm{j95gvHK3 zgf5Qxp-EXmU;qa~dqat6e3EX_=7?6!zF06G&q2fbr=^V*?|?rZ81_T!+o#x6W3Q{Z zgJ+ALK6$j1($9tp!n4(+R7EuMP3KH6GQ*{V5BKi>0srzO)Ra$ z%-+c7?~TTjr?uATy1;a0vm=+uX0z~8Ma$ZhE;fgS7>B}-C)kSndpRUxp~&fEPN@mb zm#G(8c}Z@@qN?;zw%j>0vTjl;P}MqDGns7rHmvu>lY{9Z4Szj=StV|;v0~HBuF0^{ z*Eeai*JLyu;Tw+gI^U#PBD+SuZBtb#SI&$oX*HRz#D7JdCRsAJy;as+&oG00S9;g0 z{@cB_=^B2m!xZQD2A<~W8cfB1@=Eh}6_L~n7%K;8mWV&!5_qBRZ=1G|8P?0YC*C*# z9}Go!7nfIk@WAtJ%NiLlRQ!FRBo=dHI3#^jwPOo*M;U{68V0!7T_nYvG=%88enPo}qcx+;qdA z=F*haoUs{ePGx54jd)t+$jXy7vhy;{t#Y6aX2ciWf%Rb;_g9WVCeHIjIEo25)3xeK zy;n>PlR&U99voyRd_34MQ(EnGQXS;GBBO21*6_6DH5sRz*@kGQLo}yo`EAmiN8sf&2a5W?Y489nmLW zSf){wK03-Z31QMGUsGzh2O}!O%;!r7ys+u*FD&>gGUZVYFR|`sW-AK6KrkHKgNd(g z%Sr0HAA89JoT1rv)716NUWjn=Fzo9+4PEHFl1~=I4$n4>rskHb{n?DOF%)a7E~5z* z*BDNpU@Sln4){^+K|ebM_b9!T<5-a0z*!xuxWK{SPx{bgiG0RkdI~B1rYv`xfH`YC zJ=fe-SBEsxl=*9J8nbsqdh+EON4&Yn7}jB72b8IPpL%=TEQhAB zqKxNFcIq(*GZm}cBI(^^9&YF{9Gv+2hp=0;$v5TT;a1MkOx$Nv^U5t>K6ZoAy*W-o z^%>QM++#YIRLo{3qUBhhiws$DPIyhE?8nO&L~ z_Ghfl(|0J?rB2*ai)@yQ_V2Tda7GJDJAHl1HqO!wn!8alC+HN6rl}wH6^s>DeQTtQkc`1 zoGbTfmVA0k&rJ5xLos#op4-ySU8kH`rmBd^@XwcCxwX@#`?Nz0LPh?+Jf#&8L zEa6~4dj+tQoP1j^wcku&Z^}1_d_xIblEdIJdPI++tIfVN;ls@aQ1tHi>5U1FdDT4* zRfs0GZDSMBF#7^{1Bg>irGwI(RT1oPmh&dLEwW1aZX?#6^{A?zF1V{%q1o3 z)0D-TO|Q10a!5Rr%!B^xaepIyl)m@EwCrkbLEv5WyuAxV2mPpD5Z7Ax&Z{<~*VgT- zhqs-N%R-R=?irf%TVxo-U6O!5>G#EwalSd{+q-t{X)&M78YhadF6_KfxaNI&a!+QB z7FBl=^iFmUAA0l#_n^XiFFGxur)rTn7=mvwg0?t4uBP5F<@9Ag)t;wYXW6AI4%}1t zU)qz+m&E=^pD*Gc%yH)pgFr@|Z&xY{F5L!G<5b)q>;cH1U)uoaJ~iN^c~7qrQ#G&R zsv38uJh?8f!%UmfkV&15sB5(|@NQ>hP%GC=eAB61dZRbRwQh6;ua&G4)ou<3OERyI z!M8WpPEKua%=xBbl0zM8-)V-4hhkMH@-Ug=vRVuF>@969U?hrT0XdtP`tqE**@F#p z(C6=G4=Q}c+@V_Cs{3!7xVxggdP|`?JAI``qH|maRqd*ev}5s56epXSaqg?#EzOrV z@2WS1I4;%gj2mKo!9)T#D}#egdNJ_z;tVx)omuHhE{d~jH=r-MKNjRU#_hmUKLK+U zm!fbq0CkzRH&sn?u3X~7ll@zU9mtj`v$5HOsR1+#lng7AifX zcnTU@M+Tw{e04K0rvY_&^Ue@Vv22yW6F~AN!ogrHmtg{Blkb{>2&zml-~Lb#_wI9P zD=CFfO<+KUage}m0CO)j)h1tKz`~Z!HZ|R@`gA%fiP58)FV8+L&-~2TCfW8VIp_A| zxthwZ^%*lbokdCSkxZ`BTA9*W|6x@)6geH4RsYCVgfAMK?O2X4U2?S-_wMk3KiQNy zCd{gmUZ@mNK0UAlhgN1qQ-ru76vwAFaozk%`A?>a;KEHP02LihhGMvhBy)>3Rqh;v zAzYAyQrG(<{)lc8Q*2!@wbdQcY#!Y$Ek2_WQ`-*QJi`^wXoP$|P9MbWZhT7Q>l=(E zhGT1U#@1!U_6L)=x*rH+Q=9ml^a-{ObLLPf*?B@(S|h$>+#gBsheea*jrmdWEV)wS z7$3$3f`P15J<5ENhTo}phQCwQ&woFMZ(3i#_vufk3_`-qLM=;!!;8+5szljFMF%`6+9 z^#T_OED|^_bVmgq5m@pQoo~Lt)dCv@ZV`Bmz}E@9QQ$iTeo)|p0v{JRF7S^6E1%cp zoiA{`K(D~p3XBVUlfZ)l4+;FZz$XR%QlRUnn!hE`Bk(eT+XU_u7!x=m@SwoM0zWHo zOyF^W|1Gdg+R+kNFL1NKUV(cB9u#Q$@`o5_{5&l1gp7q-1n&`8CvcWPlYc<^_e}!t z5O_%7=LJ3~a9rRifn_{?@tiNvD{w$yT%alMnin;_!2^OnB6RQ1q8}B!8{?Xv%>pkI z_?)DFOW+ELZxg-?1$PR(P2yt$ZxmQ3@IHax68MrpBkv!joJNUXAh1f{aY;AyVJUa7 zz+Qn~p?^T=ZxeWLR{p!4m-~F}=DS;OGTtU%=QPQN85Y8w*)*+D+(k14SXB6cmF}8E zrEiF(n5S|u_4_7K>w9R1E7QDp5)I0o%#H$)8B0*k^w1Ic}D2{ z*&I8AFzTYihKMV&wQFaqcWu2#eJGA^;al-pUvRVfQjcCP+y&coV>I4{H7*?7-pe0Q zwC|&3gF%srw{JNN_HEur4HKfa&#IQe5(n0D>_X$j#6 z{(*b3Icp9Cu!l+DXU_|^z<(z6%|uHTcc$KottioroCB};4ol0`z7JIU)3F;v;ovrG zZxK6MuqEi<8*G)^uwD8xEdG8QQ@83K(sFhL@aGg9?bwIMCsFSy=tzBAlAYL66ov;2 zt1(KuoVm6~_J-oo2;ZLBDL0s#=~IQ>;RG6KCx2{9kGMM3O^aYx5O-|`u+9?Q&lJsU z?c%StQL7pbspMTm4{$ozYYv}ejGL7P^2^p733TC>Xe-|o#xHrdq0Ho@rua-@cYJ@& zYG38LYA6`r?+wPW85v|ZEeeEH9O>Inmvf)kdhuJR5w<@v1|Ajp6^eHG_rfmTG>~@^ zM!uu)2cr=r)M~BOEh}@P;BE+MA=1XYfJ-rfm3#d;>jU@ct>OyzE`403wK2HrZkY6 z=NQZwl0~mSo(<|FM zx3zCsSEu!hgM6*~_907jI8|f9FOK*)dSe`uSWofzI@nI+i=hhqeJp-!#OH%{vuzCJ z?kawVBs&HB477JG;yb(9p%X$&FBdAtVYl{2mD`kmYmKSe_QN|C>T89T!ta@AerZZ6 z$;Vr8{5liT_x8n;zPu^|3KGAkf*Sb}v0z`QKh$Twe8%4_rNQ7JCJy13Q1I5rlJ0En z+_9~@V{3a#3m+h;m; z(O&SR(#Ao^vV(&ALMRb`@e>D>kUSLa#eJDbAAbKu)io4_yKiuiKN{$VKO>UtrxEIh z+arNrgUO~y2t_xkJq>{x%pPc=2!1CFZNhyS2@Vh7 z7h6(Z4E!1lZXXWFuMGJ3_e#(!@R0V0_VD-R5kHoKVVu(J0grWr-ZP*5nIv#G8W?}^ zC4&NAd7$p_UgNtGZgcFQ#j7VN+zx8@f)S>_rw;luGMx=s&}KL<;zm5nVE7dk{whPa z?0m%PH;h@9k1K~?y-B_Mg7_^Nv`!)xfp+9KtlNa&p1~XLVWOMK9}BG-Oss-pScURd zApvXds_@!XYgftIoJBDZ^v9-3myM;JB#Q$kR46({x~*O9RK^zje-m_MRmP5cCa~W_ zs6sH$eU+{eJI!kN(PnrH!LU$}cabCDdjR1G!r1S086Ldjm*&)CeuuFMSPb#}i~(}_ zB?u;cDNc{(zDygr00?++fl>in8(!V-aizmem8w{pWtR> z91)z$<>wK>@pFml85P_T{BgmX1RoQ8MDSyRtIkH;a|RayjvHJAc*Wo%z)6FP0Dm#K zr3)zeqplA%X?(81HJbW&k;I=ZF&@D$vr;4(FNaFpf3^a6i2;FX>J8*~QXP}|GPw0AtZd~XLEY!_Rj|$y@(2YpB1{%6Cp^FJ! zlkhXp&>a`LeM09EIs?u8J|%RAX6bPd`-ZNEhy{yDKP4x0|K2BbRatZ%p<5($M}+Rk z*E9JUI*-sjV#>`!*CuqMLg&dtw_E5Q7rKeZa+ezxx-lbP7M%=x8WFmujeL3N?hv|h zBVQi6`-JXWM7&h{EfBhrP3SE=XjejMxT*Gfb=r#-8p*(aWLgy8_o;-9P6uR9)XXT-LMCf{q ze51MLds^rQjC^_MP6=Ji$d`w1?l1Lt*=OX-L+25?5hLH1bIZ3y=x#Og<)I4*-2o$C z9=aQa?k*!=9=bb(?vRo1OS$DcB6Rl|`SQ>`E_6qXe0k`O3*93|zC3h)5xP+$-xqVs zx8PU0KOQ&o<)Nz=x-lbP9=dL!ds^r`dFaAIH!gJexzCLGYv#u-LU%&wMjy>hcSz_a zgl;4c-J?QxO6Z#M&^;}5RH^kJevmY$d?$sjMCiu8kehDSE4rPlgl;5@PHj-2V^#>= zg8$3j*}%t9)$4!q(h{J^0tEsVS)o9{0$V6FVv$9n1gx@3fv82ckV1e!*Ag&5m8BF2 zQe~AYK`Kt$v`yQzU9?Kns;d+UT6L9zQ7bH3HE4lFipG13{J+o4Z^&fb6ex19|NC+J zdHQ|M%X~8|CA27T0KTu@T}nTU;WjcZ4|SX}_PFEH2+$zTT+CC4=^j5Z7pN zEf$v=A+FWpQWh5*A+E#X+AXelfBE(;w77K8zS20m@%LC@CwRuFm43BgCaFE@p9swZrQT%A03#^+Eeah+ANBaf^$W#7(sJ^;leEQ16=I z^#*YpEG}Vj$r0i z7T0fag%u_3vkshUaRrOZj1bpsaRU~Y7$I(s#T6|sGD2L&;sz})-(9|a-4^FW{Oez) zBrf>ga=pdXSX^R+xV*(hEG{xa+-8fbwYdE9;q43R9k94)(7qAk1}&~GXx|8NWB%fg zgILhM5#pj2S0A)*S^4(WTU^}Y+>$uQzUZ1}ag7$293d`caS4lyju1EB;+ib3uylC) zf_l3xE@^R@(l~q0*I_eHK?^agma^ z;O8a%78eQ1TUb(WT&2DGrO;s4W&1fzG3}QZd=DLb&m4S@oce*kJ@_6t_?|cWpye~h zj79stGiyv6qsHKS&ft5@m%pd9?;&&cyj-@I7GgJzwxWUhq9#A?LTpH71Qw zW5K?Ua*au2)L5|ZgIr_M7&R8`dluK2G`{>jjirO{S%U9TUjClMz6S}u=Lo*X2)?HX zzK2NL_X00}k6`KGdw`ce=iBG_;B&fbpPPfv*}>=N;B&HTpId{^xxwdH*FIM!jKSy3 z;B#c~IWhPgn6l4#Mf*Gzd`?SQI{2Iwe2xk}Ck3B_g3md@=a}GgO42@G1fMg4&k?RY z{|BD~3ikXSJjWO8c{z9vPg**7jxN~qs%uOdqsHJlHFypUo->2z$f7;3WsNCg%oseU z1#U`N>tt{p46bt- zyN(4v{|tV<8T|Y*`1xe;^T*)li^0zigP#uuKmQAUz8C!bF8KLe@bk9;yN*`B;NOpG zjR|AhzlbAUiQR|%{z>)uj6du%@)*%q9_WMa=%Y7^eR^np3~R-PyQ`{N`pf-(Nci_r!u^+~+$>6>U@6!B@H=-wTq)Jw9OdLyruC&_REPbF#IW|N zUM=DFM8>k8sA$`?yg0hk*yr_;~3HP7s)e>%x%ep*Dhw_wb&sA;w zrmXj3cKKhRQJJGrLFMo_)6_4?3_#KCEd(|#2sbea<=hpnuKBGZ;f}c(u>od6iW&g{+4qEZU#z&4a zi^#RI6;p11?L9K4#hB&&;(mPg6|M1jP@+$-ANQyKPpmb>1l9a_TeORH@tY1_2DT>JP%+iNsU$*XOn>d^U~r`#fnqo9q! zIj_33s13mp4)>?pr6t^6{oUy#irC+u*7kDkof*7V6ZhUjQP@8%DmO(r=}=h;3jVgW ze7HTTS4+4(`g_?$b*|9gZ6mG>_GMnh1;Y$f;z+PQN5+@NhmLhphvZZW$ygv@M zqt^J>%OvHbP`FL1qg?-Z^-^cJJz3>N85At#+Eea*@gP2+j#p9Tyk+<~uR6-LNA+q6 zweUi#lP~g&o@ag5wrO3ksQd!! zizr;a@&sjs%U3yCf^G7#gV|pcLBUdXe_!eC`5<$M-u_c=28G+FI)gTZ+oyJD3AV}S zwJys3+cvE$*S>##S6`9C7qursSy2>jkLn2O47W%1YEixfK4odG(>bRlh_A&*E)Gf= zy61(rJMSCXUYzOsIAyB*Y)Sb!yFY~UCqwzO=3Vl6XNK{8Sx58-o6dlS}n#3ATBO@!bDW z!M15#x%P}SwTM@S{ilA9Q&s{EYZGa;RZF-%s#lBhrSZC6xVFwQ2Jtb)%pYpG_EAf) zUDuOXXnXD^KQ=`q1De_m4^ zL7hQc;$&03T7qrz8P?@d%C>1;x%Oy$Tr||CSN(adK)FSfKtUTb_;SZ5&w4H4_Qek8 z^AC!kU@6zWD`w2ToSCUcY?sQ^=d%>$WKi&#EQ=4fO?AXr7t|K+KeaD~wL?p|e=}+uO4&ATFW0_c z2F~V^@4UPY4^7yq{K2By3M#j5$L&=c%C%SR(h_cOoU&XLMZr?8y{C*1UM}?;eC}1= zlwaOT{>UcN!KPaL(W1sP&Gp5tOwHhR{fL7|HD;+dk9O4mYGb+mQMsOH0mJ`u%gk)|YGVWwS2)r<<4dyQn-?uUB}ee5J$n2j$0uc9v{U+xjx~pTnS- zeYN+;S~YV3e_5hA^cM&J=}4+(|F~o*(=BOVP`_fr?F;HJSbVf(d-Q0(|H`%R<>ScA z3VhGFgHfhh{YCv3qs+LK8>|oNS32Cjp#Jc9OOv;NlA$unwC~(g#$R+bQ;%H{8gS<{ z_r2ubO41uk z()}grfs*uKNxEkD;r&xvlCCRB*O#OlOVUjx>6Vgodr5j;NqRv^x~n9;t|YyoB)zF5 zy`?0*tt4GNX88V(ElH0rNykdk^(E=Xl5|r^y168sDoLkH(hEw`T_x$>lJv%sbfF}@ ztt9R2F?|0cCF#17bbU!WdA|R9X9Z*Fbt-&r$B*ZJj-n`7%3Z%S$@T16{6$(OmgFbZ zBV3h(5`3zj4CT6^vb9BZ221#QtafM#x39pu_#~b!LV3!x@0?S{rH#^(BqoE zV+YM0A3NjJ`1sgqr^a`Q%kUK@=4aYC~n)du1x!x*o@CRbbo5bY%J4$UUh}0 z{n0@fv9$Qa2=bT8}P{Food%5=URKyHFnm+p#o24SW zMeR#ab`pi#r#j2EPwmhWZlBA#0!rI9Z7NVIV4>7&XS>|McP=M;#x=YnsYMj*w3szhE4TG`w;impg6ee6Vp)X~)^me^feva^d zPjdF`8P{IG)U5nHcTFfh((k(k$L0STW1LWFR5PL4V}37Y(#03yBu<_vUc3p#@nSzG zO%q=HD{8@uQ}=V6HoTZd^YIz@$o~8+5-;w>v*{pS+y_ODuJ(8{K8hDzGzl;Epa#77 zBASL5cRPT2ck$u}P#a!!(LB7ELmTm80d2yI(F2)J5-;96&T+Qj#s5amG1VSF&WSk= zFD^h6@ZwSw$BPf31YZ1aG#xL-m@G1d7e5i@Zz|wL7Zvd0uTk}})gDiKtK*Epi!K_E z7jtL=UYvIbeS#MUP#iC&59L_k#rKV;Pw?Vu)P@%mhjA?M1sFfvaTeke@CT?HFaG98 zX643<(W9sXFJ8=N-*K^Oj}PGM@#5Pja4hiRxu_X0?t2V#IN-(OQ3qa}hZf+)9O}jw z;1WK&Y{!ec#2AMYt377%QM||>e{m+@#S11en=xK=Q7c|dp1^+L#qlRHZt!9U>cxxE z$^4!GUTm&s4B*ADqV0IG<{gax<9I}YhokX$aSn>%#Vl&Xi?Nfb7cXYt$$n`)jGw}B z$BRjn!;5#LKD_wOIAanoCQuRYoa#95K-=-+5>$OW{R4|=9A13qX|xS5eiBW=i)l0s zFa8BJD;NFMb{M;>F0jX)|7Q-$Qy5eGUs~9Nsyd>jN5( z7oS2gy!aeyz>9ibODG;5gOYf$54GUM$Iu+SSU~ge;t6Lke(>U>s2eX9P!=COlW~G_ zc=0O<%EgO$G>8|!{Q<_(3Dq7)eUNhpFYbRfV+k)Fi{{|P1e%8zU9~7@jldy7xSnMFMgT7dC`d%t3JZG!i$%E zlzqgD*P(vAcncc9i{md~-|=DsMJBT^ux$qQ;>DGy882=^?RfFn3mGSP@v~?lUR;cN z@L~?F$44$AAKHi)_iv^wyqG{+@Zv(W6)(=4N!#$^?Py$mwZ|3sI=r|R)#Jq+YQT%J z7TS*&)2J0Mu0nJ0;;xs{e!Lh%or;G~pf0?4>SgpHUVQ3Gjw4bF;W z+=3sA7oSFTc=0(@j~7o!F~srWyHOKfyco6M#d)Y5FJ_g-i$CTsAoSzKs;fATc<~st z9WUOF#+*c7!DmqvFFI}PC*C7Ij*r0%O5nxwujW357iXb%y!Zf`hZmnfop>>F4d)tO z)Li!K@Z!lI=Xl`7v(RR|xEKxK#qXf)c(I7ayo0v2^A`$GEnfTxisHp2nuHfWdM(#9 zytoEU!;9x$M<3$F^HDoqOrZ|Ec>5=4KVJMjT89@;`6S~LFP?!m9$z>7|rXCu5=K-2L>n7fhl4li!HiOqO% z+|7(nyjXh+ad`1FD32HKxs`JYFFua8;l)!sX!FU8TX+Q;ix)@T##1XaFzrp0hKE=e=i#ccPuh zI~j+N_oAIzyvV!LP82Wlp0rbk7kQW3iQz@wvv%t7BJW;1alFWT*-j%~f z+s<^n$UEFl3to&bBoz_i|!0)J#<|90WYS|m{Vv!ETSl0lGa1scXt}_VstU%A20G= zJkJqaD*$)4>KsPc$i0xiigZy zz;hDuBATvvm_f~ohs=B6wBkkPLU7vgBJ(3S^YCI8&Bu#Hv;Z$Mw}P`!@sRlzoG!(~ ztfY8YMC%m~nXAFsfESs+!P$rxUDS^k3up^oWbOy2sCbw}gNlbPa!w^47ElddWG)FO zf)@*Dyw*eJoN(&#qKg{wVgWTN1~OlTlf-+(x8Re|MQw_Q%yr?U@gnnII2px2=EQIo z;zj1kaC-1!4z0(F%%|aO#EZSu}tbi)b5OWPT23yJ8@7b~t0+#d^p*9?m$t z$lM;z1iZ+6AI=oK$Q&R}BVJ@)5N8@*WUde=i5Hna#7QXzGN*{st{BKXBhEa$$lN1N z1}`!niPMP}T}kmUkGk=q^9}X~FEYQ0(~B3G^Tf$19x`95vq>?Kxlx=gcrlHNc(H)C z<3;9GamJiRIgq(loLaod{436QyvW?NP7E(HPm5EJ7t=~B9_COZUUa_A{wp3b?~Buf z7n2fRbWsalETA^L$UHL69K4uB9g2r8$|xQd&_cyS=B07E6c3rJ#_3T!bkRD+!vb2b zco@xbjwl``QC{)TMVl253ur*`F#0|EP4O^^wksaGsJel8SU_X&BJ=P#wRn-ad7SZh zF^6J!k-5a3DR_~2eVhiom_}3aVh&Bii_8h+Ovj69)T$VmMRV|C5zWJku?M-H;KdYL zfETl9q2gf?btxV)XOYvRc*s0P&N{`zEXpY!Mjqn)!iy=iSuv3LI-IR|@qV-oFCNoJ zU*W|w(3p4gYy+9s!x@VgZ$;zr;y=&?ym-LF^bcNS{tu@CFHT3(@ZuvVi5H(oZFrG+ zLYy>SJOX9#;=QO7FMb1c?cA8qR%`^9wJ!s8ca8hq~}$@2?r>c=0h*dpe(Up|hFx;KkH$DF-ip26f`aEb77+ z;V!?U9K3ivD&oZqavHf0!ZV-ZK7tpoKojs{22D~7{3@D)7av0nc=7LODqbA-C&mC? zJOxe1iz(EM7a!k>KZEB_xE(FPi_xd)3%oc7ZO4ndJi{?MllH?H8jlyRc$RkJ#f_*D zFHS7dCwMV|+VSG~s2eZ-3}x|R0p;*w>N&<0J`KN(@_6xGf8idD7n5iZFYfz1efVC! z8-X{W2wp6pI=uLi7r7?m#cNRuUc3vn;l&(E293ju zdDM;E<63a1V)x{(TJ5?&m)SB2Av7tcl0 z@M0P@cfklLrJ`tMeTU;dDMv)>-MLg@#1A@9bWtq%HhRGEyoWpCedcRm`4M6 zF@6C3ju&r5+wtO`QO#M^9$OEj@9<(4)!~aU{^km&9xtX*94{77BVLS*qo46&3Qfn0 zS=53T(+5>JZFn(@(pnFHg*x!!7L>t@XT62-h8L5l8!zs8F!}M~gQyoT{x|Bwi}g|V zA1|iRX1wU4EqF1Hw&KP3TPvJxcrk;vGZ( zKZy!>u@`N{iv_eDFUAjN419ocU>-&AV&VwK171v{@p$p8XaZh*2G!%mXdUALFQ(8m zyqHDP@nQirzKwmx ziwU#=FZQ90c(HOqh0~80qi8@eFpsw3MdlH5oDbq5a|tgfl(n33?}den&*^QapyM&3^U z;>B~&dc3$AZNQ7slh|Lpm_P-*m_$Xqcn2E9i@!kC)2cl>@8G<^i!oG(kHagJ#)}W4 zdc1hZ6!sl2CQfGG@nZfIjyqnA#X0x!aTqz3zQBt&qWO675!8tn-~BH35ih1rV;}LN zi#FoL2hk?HxJv`qK)e`1MZCBe4dTUH-p$xPhwt#=U8o)}eib$1#XL&jBk!qjjzk%} z_$cbci%+90UVQWEj6J;gPSl4NQz(zmz#J;z3vmBN&Oy959XaRnSpa?<)#1fE&{Vwm zYt(`lpGWiX;+8XL4_h;O9K7b^&^!q9@M)+yDdyufkA>!{ zSb!HZXdzzAqb|G{X<>iyVgjwliy73Xc$i0<6b~bpvOkK43A7a-yNv!pgLv@~RP!Ou zDLCzN^5DfeXgpqY(FDBs&DIKM3SK0iXzcWAkGJB-F2n?jv|Hfiie8g*x%# zFHjCI7EwQ5jNC}y;>8%MIiE2D<8#@D7jrjNI8nS9naA}IFJ6S=crlHp8SV z$Gb3(I`HB%D1#SgcF`wzaW3k{i(f)nyqJ^l;v=XJFP^!Sa`EDaPysKd(Ewgtiw5!H zQOg*^7jd4#_blhSiWe_Llknmdr~xl#&@{Z5Lv48RNi+v9Hgz+G@!}`Z0=#%PT8J03 zs0S}Tj@IGD^H)$VUhG)Oy$mldLj}B;Lq)v!B-)M_n^rNlo2eJ3&{({9I~s=9hf5ig#-nsV`CJ4)fj4wS}=`>v&5@Z#~v#fvGF#f$wYhZoPwa=*ii ztL|gJ@Z!U0D_(TI!hYe!JgS+=IDwh3vR`;HzK$`87k_>~_iMcPCo~N&I=zfTyqNnY z*K533Kv}#P{}$&IUW~8jSpr{#iEq<_*O3md6h9!7q`@xw>q z>Yq{{Ud*F0m(Vs?c!V*47b`b$-0|XZr~xm25lzF3In;_5YaV5s~agG;bPqM#wahENObG&#oYQu{OGzTv_zvsCEFCLHj@L~cL6a(Xb zAnsC*H9Y$%jwxQe7EQy8ccCO+%%WDj82>YUf)~$tp0S4)Q)nAr%>0$HeHmjOZXV>` zgcsv~r~P>Gl>g%S6YmkHwscxyqHHZ zyg2#5N@v=YJd?vOqUm_?In;_5Yu;SxwBf}Fnuix-Xg*%djbp#?qI(c+m4{nV7B4z) zVIT2g4CV0R1*i`%rqO1+c)?*DYpsW$Koe&1d*kq4RF4;Pr~xm=jv|d0ouezAe!Lh% zTeKc#&{n*7?J@KlUUX4Sit`u7kF9j-@Zv?WN@o&YydF)%iy1T>FOHd5=`6sDF_gu} z;pF4!YrJ?V+J+aeIi9*cR_!r^uf>b&Q9WKPpg3O4O=4W(MdyS{CxI8^s0lBoQ4%kv zCs#Tx@^H!9N#n(Dp>DjGM_Ii1i7C{F7r%D8QoABbE?_xaQ#kZhscrlKuuVUZf zc@34$SiG1*F?<^4Q4%k%eh=drFXqt#t%uRmsTVKyHL?va7SJGGbk1Nrw9)4r4 zJj#Q4ufRg|_3x)gPo!u4aE>!`Ynsc=3B^5?&lXjcs`GCNvE%R-MCf#EWl5 z?RfD=XdYfX|J+Ka6ECJw4_?fDgky6JeFF2S2``>>K5fN|XI;Rx2`_#Kb>hXTN%{dV zW+l9szKH7vUYy*_`HL6-b}7f^IM9UQD12 zUi{frvBBdPJiLWOHd15yal!4#kb67 zoZ!VY%HYLlhU)-c9D67Gg%=+}MZCD+9@=(2zkdsJXe>StBMa#_yck1iym$fXz>674 z@i4ZOK2Z$3e>r`P7k`R2;l%>lj2DY&>>TDwrI_<%W``phM!i$r7xz^#ub*TCi)gB+k z*W$$+zQOUri+g>OaflaVXewU(0vg1N#r5>#C)r2new*Wg7bks(Jb2MXajl1M`7XyA zFW!ricyT>y!;2T^I2L$u5G}xq(eE*a@nQ-!+`zLX%%Z7y@mm|{PrNw$2Rtv~#e2~L zy!dm}g%_vv(HD4e7TSOpD<7s0@nRHh!HWr0#EX~zh&H6FJ^p4R?Z=C|{+RO)FTM}; z;KdYLj~73UoE!O$7moP}!iz~X4lia=9li+pMHOcf-i0rsI9}}gIpYK`j^9MT;l;@) zgBO#i3ophW2Jso#wwY}=@%a>f14Z%TO}}B^@nQir;Kg@6!SxU? z-iq4r;s*+xzj!f=I`QJ4p5$2I#bbX<-{Hl-qs@3R@jKdv7dLI8J@a^903Psr`UEfj z4#n_d{0|5(KK@6>9$q|jfPTb_PosHwG5!?eQ1P%Ab>qbs(K@{N(4QEGcrp5C`VKGN zvz7hFd&Fqv z!;6vKXfr+v)2Ib6Za{5#@kP{$7su>g<@Dgiccb-q@d1>_i68wpct5ZQ}L+jz_y{epEyck29 z@HtpS+wjibv=ME`i_Shwr|~J)!^2PnFHS(Uc=2X59xr|##qeSt)#JtVSlWyiPu!O_ zrytojx;l(eZd3f<& zv;Z%D8Fk^sMQ9yfT!J>>#g%9iUR;H?;Kd%a4KJ=n)fqlF!Co{LFa8mY$BQR@nQ?^| zKY$wX;#!o%icC*gf4iWi^kqHTEb#HF+iFV03ycyR@4!HYL6V;tbcH0r>MH=<6wI2U!}#hXws zUc4Fg;l)p(e!O@)8o-P5(I8&@G^+U=`wl;YYVl$Q)#1g@p?bV{H)_O-UqnrK@gCHI z7Z;*-y!dm}ffwJpoc80z$*3DIX3%=P_>pe<883bU4dBJd3fhbp$FHQ#ck-MDk4F>m z;#p`4UM!-icrmt`^ARsHcZ-w4i)qw>7jr0s7nwiB>BKY7jKka?j*A!bs2eXbhmDiP zGuI988qxoFG5J-}c(H)`@FH`nIC-sy&eu3Dc+o`#yy$$L{ndJyL`B6z7Y!-~7Lc=m zGGX+7`T;L82aOZKi#b$_7oBethmU-x%6SLsR8_apF`8}+hOg4>>FOZ4$a4}gMBD9pZ)qlm2(?%@bh5~HR8obQ5@e7 zormZzytprF!PmlT(FFV)*n<*yF^8J);;wz{BYq5g6h-m<@M90tC$y&xcA^^Aiw~e$ z{06ws4;cf*OobPrK71=&k2Vt{{_#hgJNN;3*+$xdZ-p~{O#k4UVK3T(&%qZ_-KW`? zpK$#~DZDu9r?ekm1K*4q@#EnAs2<-7e~_n7@O|)vpRteldbk2L+6-P@6RG(7$?78zk+&)J8kssHKL2gTFDQ zd6Nc6$3vBA{v@r_qW;q{DAFFatLgjx{PYWs<~${><+b*o@~9unQS1JdOVF=H+O76W z8V_1O|A9BIk14+F5)ZA_+)?eJ^#8UEw@q_dscpgeQ>3rdXN?>DzZWv^81CWL;MaQm zmmXvp|Ah#Sb8w8;(atw&37seH{H0cLJ`~7;#3+Ii862Cdp@9SWk zc2b|}EkYfq^)a4NEdI6jhvpFbS1&=o1mkr2L;w2z3ECVUrHg#Y|9Wny`_Vr?H*8?t|MU_M zRd{iOKSo2V{#R|dZC@{GTclW>))*cAD}UbEfXw?@c73q@)$S1Ys~Z%nd#VwV`SDP>8t3yVx84eSt>x~2w%>lu z{};476)Io*nm)1GN$zC%n%__5s+>N`Rr{hp@z2k3llcx{C(Bhf^^2%7HRs>IYSFz( zWzMIaZ`2YG6|A`nYpF0ezQs_zyAqdqJKt?tyxR3@7xmA@DZigI_g*j`_kDb(RT-+U zv@Dh1K85d5Y=hDx#mzZ+__!M>uI8Q0BT!PFcmMzCQ2%NiXWz5) V=MEk3GNP>Q> z&krv@cKXhjuX#*VzUD9q%I_>GKl?h$|IBaw@>5Xd>sm4xD*s^8t&PL`+q(1>p>}H? z6D^wCL}|@~qGO=BQIytvDY}Ph4i%*}ABw*B&>Shj{&xIqs6TgOThAFg|GcL8Q?zLA z6qT=WHV`UbHKoto`QxSeQ&g_5JMo_nwR>06^WV$Q#h@0|`)U{Ui{?8KH3y2Qc~OFT zxhlR>9JP-PyFr7597s+RDX6~Kc0Doyg25E*qo(Yw&ao&IK&Y!1x z2VRTbiJG8miSng}BFInri+j)M(65j8(7kxx`7fI+r&MzWuD_tKx zzg2$yZ-yV|=@;$%_hEWpS=SWR8I04;Q29ECnrH6(d7yV~wdg(Dp!~s-^4l-o`S$Bw zSS@-FR^=T_M& z>nK<6`>OoMtbFxN?6*Vxp>Y*y-TD5|d%$Y9p0PS8SLLUEH_AB_PPu~q@DIyt9#49= zSMApOyH#I1uJFLFa`^4HOy<@B~rT2(My+bVO{o#L?MJ>^|(b3R8 zj^8qLTuA;~F7Z&S^v-n$I}w}@9ijGXzqVYt^VciA53O=lP9apTzT-&D+WDU&>m6z> zdVgBuM(j1wn<=*rY6-@+pW5;NYOmhE7WGbc z&~E1s!`t0U`7tZ%wU)2v(7hj(6sdL&rm|mwwyXW2jvcyl6#@?_|wpF zB>9Fcx|iyE0$s=S-h3a_y5M^Py)&$YlcW0j~^)7rcE?Ov8?aIS_;AY-O-`R1Y_ujSW-S#(XQOQy;E*iED^~)Po zr^Y}qF7%$ier_8c7kbBE>*ApX=w1Kt&v)~ocItlFeeKTQcQh}6mLI@ie8ip}>WAIg z7UezoMlb5MpWk{{U*)Qnj!?O3_ZG^n+sShEj=mPXw;znZ0m|3-OFT2m84H_ux4lI{ zEL#6+7hRY1UcDB*Gyg^{Dp?A~-v-+KMlTv`8h<+9HGe>TX#8oN<_^%hI6hc3hk(`v z{hWB#uK#dvg4Xda|IVK0nlnI)<^_187L_ao{XC)gKiAKCuF)I`{ZM_c>%Z1%egv(H zqhPt1KG3@4QT}sU@O?`und>L5@f@rV^1l*`Rxb1349m|PI9}R0!6$Ry4BP%{>q#E% zw>v1GIWN3&g7TT;!B1QH%t7I&7y5jq_9u=RUOscL3@blg(thT>@wPMHiN{weKW71P zqxv{l^Ax)L)vfBG|7&s9f#g zddl6|{nC6FBkk8%<_`$pucF5fKTng7`s0kb6}<75vUKSV<#sY(deY9-7eB$z$Gi_- zyE4|Uw56Ftr0n@ouzbuL;qT95K3{47^cf+7g5#5ZVQAdR#hFi_(Kwo?)?E!tJ(eJBVh)zVOqchR_k;a{P zp7DG*8R_|1`p@!zrU%mhXZGYjJMx-(;Q6S3T^%$!Z}ro=8jH661>=jx(YF0=<2lCP z7!NzdFDGTZ-&k>|zkaIm^A`UB<1dYGwfI)!SB)@7{H?~H8OK@vnHKj};|k+*#wiwmlkvyKTFXDf z;tnz{H9l>uxA+^34;#l?{_`#F0OKO#Q^rXaf4%WRW5n`*$l~@ke$n__^` zZM@g`d*iW(``fQG_8Ip&!e4*Bai#IUjPI%Qw}0Na#dy8d`(5K`tLOd3yNw0oQC8p8 z#s`dJtez(0QsWE8Q?0&?@d@KKR_`~B6;{s~#s$V-8xOPkQpWEXcd>fTHoC^5af;P< zyYW}Xk6FE6Gj2CFSbcXGA2UX+zRQi@FjiVU?>F9K{IhYg)pv{WQDdvsmo@&)c#75c zDdW$KZ?^g_Hm)t@97!S4jW*fg}Y_j_9HWrL^R^L^|ZyKwu zp0kW!GXBAMoYi-O@u$W&TYZ-qzhO*ReV;Y<8>3d=<;JfVw;NBl`aW-b+<27L_i^Jx z#(k{53yt>~8?Cb3@SbejM-!<-T^?lg5!Wg&uZZ!58 zBUaBxjEjw1jgzgun~gs)9%%JlV!Yp2W%Yf~xX5_-hy3gE!^UTgdwkemKf!pKvDx?u zm=ySd0zsvGpXZbI%{8KIev6lZp%fE}| zUuXI6u>7+v|9O_b!SWwr`S-T`FBy9*|E-q)GRuFK<)32t54QZfS-!s+yDa|=mj6P_ zf4b$LWcl~E{FRpPS>r;>|8dLzVatDt<)2{r_q6=~W&ES%UtsxXS^f`N{>heqyyf4; z^8Lm5gyo-a`7g2jXIlPaE&n*n|B~@fmj4%)f1c&P!1BM#@*iRO$6CI>8Gmc}e`@*X zSpM@Y|4EkrV9P(o@;z((mF0iP^0!(3vn>B{mVbZCUv2sRX#Bb5|E}e4wfv`B{-Z4a zo|b>R@d?ZSL(6}^{&{w0?GGnW5)%U@^tUo;jh|AUtQ zE0+KBmOpLzudw_FTE1tEKezn%TmCM~f2-x6ZTT;-{9`QNlg15}e}(1$g5|%~@;6)l z4_N+x*md*~%l~!De~;zwu>6->{&Ow=X_mie`5(0W-Io8;mj61-f4=2!wEQOo`7iOW z!#TU2MlSNxNn_UNH2dolM%P$0#%KE5Gsc23cCo)cZOj{^R`2pl{q0{e-f6tec!Tk3 zTYI$EO$RDiN_w2DU>$P1!_ejP^JKOXF zfRbff6%)ER=Q?LQ7x4ePoe$#9aL(d;^ zZPj~P%-w80z%=UPFG4wO&TQvW=St@iO1so)b!JlXmCnV^ECOR*OpLbs|0eNGH~Zl< z@KgQ@=QLuja8k|;;;*GOGf2(!idPA3_*Q;OqNv$K>Hit5y%cvPQJ-|)Mj1h!4Mbn% zT;#PS6^i1|sJvWO9s7SIB~IgaIA#++!)dmbN1b;$S9`4wuW6;Gi=4|TH%dz`rR{3L zY}%@&k>3azs{dTdn@wcUI>$MVJ(VVq+y7swiO$=-H50w9$5YO|&dHQBmHd}bdaJj; zDe6$KUP4RTh*2L5Ul;YtjyWedM|)1?Mnn5LQokKTRMa_@depZce7O$o;|IMy)P7up zYj%#&ZhAjY3b$_>+tqV3Ia=xs^^cZ%ejA~J@#ME#BSL!<93Q3pquWM5UP#%^UK?~A zk0oD%e6v`0H9g)o!dQ4c$4}#NHv4!UdweDJ%yK?%?Vjo#$Kmyt9?e&KETi;;zb}{4 zhZ?OjLnF(fe*c_FvHl`jq8=I=c{(q&{an`1e!1r~e*LqF-&f~6WMXZ$>YTZX+~G5V zO-@z;|5^Tj>45_C3DZ`0N%xZ8CAlSiOY%$lmlT!^EGaG-T;g=qbVa+IBJ(^eW?A>L z?6Tfvxn+IJ^2_>{rB`&U$gJpG;jZXjkzLWd!dVqr6#FaHcQtmoYr5CeuZ^#5T$@{>EMm=_n+ zEv{eOxVUL?%i{LM9g8~`cQ5W;+_$)&k_Q*pP-fkd`X!A^nwGRIX+1j^>+<)4R+NmtzBBTw0>#h(x#;?OWT)rEbUy{ zy|i~}-_ri214{>&I#K2>rVZ`OGRrz&-fvbE`Pn%)qnMj_lZ%RrcHGA3lGu`j*SG3T z^+C`B3EGwPT9&45nJ$;sWxe+0Xgyt}bG+1=8e>h9>yba!@V zyL-EH-TmE#?tyM+Ma_!HinsH2BHm*#pY+0FF*}gKfvU8=ovUg=}W#7ue%7K-|l{Ji`TExuO=_9S|mdQv^@J?Wm#9=E5vC)d;0lkXYm zDfSHZL{`_Xj;^j>9betJI=Q-Kb!v6T>dflS)!Eg(t8=USR~J?ftajGatck3tTN7JT zzb3J!X-#rX`#i-4!?PS#UuFb9O zTU%H=u(rtP9p&(AZ%&OHackX}Tkpo*CO7G}xM{b;&A8oe*6n5N^t%Oj&~+BoEQ&6w zTNGQ=xG1rxX;ErX`=a!s&PDE`?nSvpeT(vo1{h<5iz16_7e_f8;+zjj&W9A|LxwXV z%Ndd5jL_NUpJ{cBxO&E1(~=}-MVj+MXLr`lZU4Nk_0B|{g$YJrinAlb+2Jz!a-1Ip z&W|GJM})B#;|z&2>XMuzY0i-hBQDEXlIJWb=qy=Qvn;x-j`7yGEWvn7Eo*1IbuM!m zZ#nNuF|e%2c#ABrWxUldk2Btq%Uc+49m_L}x9sv>##{gL0^`l;uIY|+*LBAjZ;9?E z##?)Ln(^j#cQfAly7P>;V)r29t#(C}@fKgv$arg6kz%}Myem=firk966@?W8jJKMV z5yo3=Wj*7qX=Re}mR{Mxc!)JF=+OMtA+9xQ+1Sq81zF9g8wH(zAKL!zU+`%sGe zn8Ub@F%C1!`g6B zbMCsFySc@A&fKDR=Ek}2x?M@mD9#=4$n?>Qey%ATllUlSKD#t@M08bY9N}p6(k}m4 z_(!6K){T4|I_wzqa<%E_YBRW~hQ1Hq)zYDRFISn|DDS>lurtHqN*w8-am>xN4dl2( z=eZ&k?cN{dE*u1LCu^w~9JfV-A|1<@VLzltR7p5s?3syz5ADxvnF%-gPAIT}N8H>qw^Czhn4kaf16r_)Jc&8Ge5K^LZ8J z+CTh^atfoo&nLmy%nZBo^?O%7hdZsq?`Nm!n-<1SaOMQZsNniX$-fH>ohRuun9&w;LY$LlToL_5@v_1<%6 z(3dIhfxWo7aP4NvLyG$l=n>5@$yrd?vc4+W8A-2 zMtC-hdiTrlXb)f6hL8EY_q;an@<@u<_>bAWG{Li*e?M*K`OUwlx;(>W?R?1Fy|u`* zoa5bJYk8jY@3C>7=@NE+r0ias;n~jh?zg=>-}(340?&BbT7{H>{sya$Adio)p+;hD9?cYeYuh6z$WkBoZ?w9?cJX{c^>rd(K((8 z^WJ@WfagO0ULE1t(7#{T^L*Io-LsQCBc{Cjb_dUio!-4W%d=w6yMOC>alpHW`&X)3 zdv2`v?&S%d9g{sR_WT&W9{eA?v-6>4wB65!?j$kq3AOQm;ND)aXIZX8|5^SmdLXSy zG%NYLg>w(SYV@v0-O_l=-}b23rP7^ya5Jltl@%2e_l)f^de`JFjTJMisw*o-IkEQ9 zyS;66MRmp8$(0q=?q#u-*zut?d-9|D3dfBueoK0sGsXLN7GJkr#TQ=sYF1xJy;J{U z6HCgij_z4?U-hU_xpUrj;mM2tzW>rQk2(2VH@)P_uOD{kk>7u5|2qoq+&vm&b4UFx zHn-{t{xE$-W#vBS@nNBeC&+-qaq2ak-N zJ?rAOkI$GrGuklonwhP$Qdi8pvMrjJJ*)ZZi`uS=&8;{j+-n@eTKo3>PW$EIKRj;l z&8P)Tmu5DylymYl zjJDVzFXySaWk0L;tkcd;ges1WQ3Lm>sCcQmlAoIFuGF5@)m4?dq+=(K-qp_ieRt6w zj6OU_*4*;I{Hy2P{reBx^x}Z+xC{o=OoKKxK}bMO6czW&b0 zjsLG%s}755Yxm619YaVDATi_~VCe3YloH7yMo=0gln_t>ky1ip2tkn&kW>lj5D+*j zph$^=)ImZ4xf@gtp7`#)&;9Om@BG2DXZBia@BPkte>Lb?(TvOGdXoyhTar-D+;2M> z91h!Ycg{t%B&lZ5W@_d3l6hq7ec|f%)>+*YevHAL;qK`mY_(IqGGglVqM>*vE1M}@ z<9%pZesMO?JkcmwT#!FTFO|?(xvB2P2j_iD(=DZ6!j@1)ZeCP;7REsFcGe|Ja!N;K za332}Xnv5CZlLK0^Xz$yEOV^H(z(^ zH;5Od8)d$-bcs^5dO~Qm9OTlS8zD;etbFbp2mESdfk9Q)TNz{4WCohd5g$d8xEJzz z5bL~03c^h*^AB#5yUWalZ9VoQ0WIdJsS2{Z5zDnb#w(wc6w~3tnUS-6?9BO;k6eW+=6la@lv`Pawh% zpWH6%#8Tst(23>Il#G!$cGI6`ND= zxt!;+MDIe&w2AKWfEb(8b#s6+Q?uwoHc2~8LP%Mc_O1{2<6OO}`|yZhG@9N{_8@A*_3bY>x;puQhe#@b;@~79jL^j1-Pyw%eT)QD4fGAv3=V)@ z4|5*l>x4!M0EmOpxG>Hi=ZsC1;i^vFz8GhW;~`JLmHm8OJ-ji#frnIpLIR>vz~SqF z3Xp&VQc_e5C5@7{`oH0)2H%`k7LP^HB zf2u=9Yr7yitE;Yg4yLmSw9#xWv#mcq8BZOJoBSe>biQoD3VA3Mu64`Nl1zMoEpL!w%(;5&y&4s8ilizv$#Ak4QpvA9WEYKE3Nirdw}zX(Ay@n zusUI8QzIA0)Vzy_VMfbmD9v>!v7|)C|C8>FL{u`PLoy!dr-8$W{!OV+S>f+)DRMjyu>g(KA@i>4yYi!X+{c+8U?FZ%Apo8!Z8l{=cnau^a1+*-+z z*b)fd5A!lCzoeUyJ-DTIqQZ_TB|ior%3l1*cx@o#HUC0sjpdi2@?Q(r!MkI5d6KAykK25SR802|(+Z~#cdiI3C*z>H-)sthd@7fVWZ?Bp>; z3?kzpnW7qT7y9nlF~J7?jN`*@sh`Sjy3pK+E`E05A(U0aM&yHtR$nC5R%yEI{?{m1 zM8$9ja+;|+xW#$;u52gbL>j8h5pqu*e#?HGO23}mZ8#nM+%~q(9j1Z3U`4JlpRibq z%;ng)@m4k%Be6z=Q(xS~VrFmlvM)&yG4k5&6{iS=*~YmcjSpcrrGt{>T(oMZzcNxv zx=c?ZgrLQ$e8Odd8Y(BBo^L#NO}kJJJ~`)P@O7%?sn_VX%|@1UQl%_<>O)yc`e}wI z>vNBdunNdiDZ=tFaSM{gFFOe8p@#->J0#>He<*9%@2XFi8SSWfllWEGmAMwH9w}Or}ya>|xjA)p<%UoFFk7 zf)53+0$;y2>A&->()G->$S5~LuR()k67U}u#fe_{G|@6m&UE523#Df#v)rn>nxIfi$`ZDP7fbf|A6B7DI_jb%|g7%W}$j%YtBFT<6Ip{qBmSgWokrbeC{ll5k z10sJ+#6xo8!J|{t{baeF3Hu3atgEY)P)jac-{$H&?WD{_de^giVvh?5&;sE_KyY+| zEo&CdUUE0-oGr984=&r9qXM=>EEZb7u;pr>DRf!lOa6dng!MPO2=+H4#Gg@*Ls6L* zkwY)t{Jbf;FU+_s>hokV-E593Tx_8COXt_hVieIDCvKAE*vPoIfgU|8x~-&X(EfaY zj7W(ru)mB$_Uf!Anfe;xx0U>+)ZF?;!wy%wl$HFRgjxQL>a^2F_+yIpQ$wDp7{IyM z?ulZ5#8CLkf~FPzxx9cePQ%G*D$+e^8=QnSEIYv_|D$+Q)5tPRreq+phSTmA)yf5zs3a-5U%fd3?>W2RA6Ed0igkFOL6Ef zA&?(SM_Q8LLjEQbc$q*iKpOH;VXhd1Zx_do;%9NVEP3E=XZ4#vUHN|D;xtAhmqIN$#T+{?+(Hbt?{_!~mp(7z%~@ zu5QJUl1Q-r2j779fB6R44^;$M9{QszAInrU#X)WQuQh9%(Y-y9ZAR-4=|RX0jbsPC zd51IH1*yGCxKvx|a<6~$>RS)znHW{HbS=zA2j1mvR_;sN>^iA*y)Bpjj3%6zW_7UZ`pr6Ha<1;YQNkHfXf=MLrS)nnLqXT| zq(|mzV~>gCbYNoM19)S+9!vh+e8$>R_Pu4bo-sn#EbdV|;)chG<_*JJ`5W!{ydli3 z!`r=I+2myXoMo@wG;mNjFLU3sHr6>w|HeY*)a-7gHTAhs{oAR+CE4wXe z1CL)7bLb^$cTRWaZ;7YW(u%MU&o;ar8FrBpf+|)@y?^&vDw2`%7IuZqb6KkIijzwq zvr=YbTenU^eFq*1%}{%y6pJS0G?~unGI5=2_q3F#EW3#NvR%SR;SDC3Tw9!&tSI;H z8t=SkbS68)aU~b@C#ri$GS!R&${v#W&Gr%z5V59ZFQBO9=1p zx=!sX(4Spv^B_VP50~r2u7`vTH=13?Y+CU@s}j`et(}Z5$72;zu91Ejs65=^&txba ze>F1eRLC&ObIzfBp}5n;&`VZc-PnG=zU8G1W2630&B_NN%gMjhnAG6b1~np>GXMkx zc;F!p@MjUne@ut59oJ!Aejfe@4dvqQ;b4EL7XMJHpx{IN80&kW>4g6}Sj5ZU`#@WQ zQtwwaOz?}kj@3QCG>oVxVEiO4eSbAI@?7uABp`Kg+w6Eq01Wo90$4;e?~%H6|5ulC zKc>sq#c8P5;yN}fi>lLgJ!5+8a(mVP1SVkS2f?Y|gc(0#DB{VsOCj#8+3&F{rUW`R@#-9h*dKxqT6{&t%T0bB44v z2UNeuhifJ)Ne@h%E`6p^j?fRxMC-wAV_3{~HL$k5X>SsPq^rx@0@B^GOWt}21ulfX zgq=(?X4%6iEP0nG4pu`LX_qQid)G6KXu0A#Coq>p!*9>BYTktSr(99Eb|dM`35zrg zZs*4E@Ak%qi{jWl9srfb-w&H>rA~S!vNFn`x{nq@M_L;*Ikvih7(52%%dI~!Snlp9H@hF=ZiAdpwYEoI zOGUeT&Y#Nm?}WI+*ccE8sZ0VTDt;InBY|UR`u~x+z;631E+0>i=UvXLr#F1eSi^FC zj(x9+zQe{#$%X*Fd?ZeMQ+0Q3qWD|&xu+CjqnViod<05TDcntv7S!qIw5FLd#74!i(nrY25&F!U ze6OBsqNtb7MsaO|`q>lW5%*WRJHCW3^lo)&^kIyxHgVdZ|y-(u18#+DNE|inCU4RGyJNSZVx0IsWZ0?eu|6R!+JLG58LAT7U+i_Cs(zwSYug01|-s&o!YCdcGfP z1_T8B9j@zhjQ)J8gBhu%aR5oomYwtoE!O5n;yE}0>Q+^eF|UcL6!}|Wtx(f zHw*;^fr%>_i}9;v=43N0cSB4*T6yD3Cbw_zd6qoZ%}bC^BXzl9L;k6Va18K^!$Ku= zq}%FCR+8ejMVE#mpI??sAR(xrpLC1WDQ`W=UTRTK&~E#BW+>j5%0V!~xZ>UK3&SSz zz*!z=)rHJ;)#qp0`PDmwmMv^UTsCjs_=HZf%=q{rVZ5id{(S_`_`Y~lK}!cYx{*$h zeup7^TJcejibbV^QsX7B_rrUNd3ubHjMMAXiLCZ%igj3JB3?QIVSHor%rc@^=w-Rxp4 zL;jsyoDGsJVXm^znyL82@{VDTPL9>HzmVl)wJ3R8Mb|g#m+3b>X5~G%ZSgizcq|F35hE0Ergpw0 z_~cVJ5MFxBSV75<;aNa<1`wVKT*9LjkU8b%bJ2Dp%g3Xtf0W5EF-LM5L>v7A-T(c^ z27ehVJyT4GDMn|s%$IZZeA5nOG9L;&-jF~1HQc7{f$3CQ*tXBq1$(LJ4y~eG>d}q| zGgr}|vI#lpu+BME0<#Jw@z6YOa5sj409Vj7(AUY0LjUWR^c+VT)}e&4nBIcLsHg4Fij z|NsB_eg1w7XU;kM?7jBdYp>T{d+mnHJ3T&+$CHb{RLbL7hb#RT=6{d!kJsZFH|g8uPnLdtD$SJx#^~xqoFT-IkY5pQ)ux`q1p@QhQ509 zbzeTSpdhc@f%DuSn{qFd2Eqo1*`!QdS8}~iF7LHqk>r3&% zahsF5P zL`H2`Y(LxQIW;skXI#)TADDTNu;cM`T!3qkf3~38@^_BMGYUWRJ;(5s-!_hBWbW!xK8tXI?up0Jla!~ z=kX-};`3aL+wA{BFZnzrco=#S@G_uQiDsXtj0X7#|D75A^71I&oifkrC-pt6e8}Tz zI&;Z&*F>-JcuozYa@6Um#`Q{EUH^pv<(aaWr>_u5N}{3)6!?$K3Q;b3X0t5FeMDb9 zwJ5h^Sh?_-OO`IV7Oxm5z>S9xe*z&W{a={cUGn7{Z$?4F5#b7h17r*Q zbY(#BS?{_1NwWPit9{dH->MA|yo6^X{nd(1>VaKcONeXnzx8%1rHZ>+$|iU`y01U= z>b_c~Ym6$o2yvZL@N}dLxb49|r`2#H6FAg;!tU^N95j1EDJ{5|!^$ zg}}#i^Iy<1Z|f;DWm+TBbL`vQ+HkN5;z23&;?)n#ub6o#t%B%H{N-W zK;rt#G_k9hP5@&T0w{LWP2kDyO!hEL<(l-a!je-LBPyk5;}x zGccM!hcc;+#?-Tp=JP5afUVKQX7ib`9#2~`Hpq`L&1-T6y*rWKZAVFGEJYwy_yAOz zp_gqAP?J0Kh$bSay?l%L`ecSBb1VoR%r_Be->BRr*$2s!jQLCb1C4rxnu~T%Ft3!2 zqL)k~5}U_ax)gd!<;{Q3gv`Rn7?hIlj~l6yflFwN89!kSG+y=z(I_y(*o`$qXH))* z6u9vqEx;0>iTPvTSXo;G^v6o=8ImzS$~W3G(`ZB&pU1O-O zIr!bFC2F9DZU|qoUQ1wv`4{2I{!h+#fQ5;YDm;(s%wQlAe-T3s$YEdi4E_VjfL;Pj zF9}?GX^;U7{txq0yLKLr40^%%XzfX7crade&GH-E2i+e4YO7`YJti{IEEUv@>Hrnc zj2wB3Cv!4BCwkCQeO!JPK+EA@d78k7+$RZ7^xsfLB%ud%;f*A!13J1Q>krZs?v)yh z7@h`WOvKQGR*#83xOsFO&`316p~riszcl_d zO0c%LL>XUO+(@4^@h~S&{t0mtbfhAZs4cG0M4uT$Cp!62%h!vq@kFQM@+D7nLV7+& ziWj8{cU(mD#Xp~=pYaXNY1YmI49sYn@c-gx4mT;&09!~iE-o%HPvI%lrzI{f4wGn&wee>r)7kVn2?g1O4 zrc0>l%Vg6P+%(ZsWscn%wJ%A|RvFPhs{iSqI{mZiuW{GE7FM`Z|Dlt(YyVjNBz-`0 z9dn4jgP=}eZcdRkHOrHR3fXJJLLFv$)QlU8F?FKFMNFBxcuyAxHSrd+h0UoQn(sv> zwf7~di5qFqqs3vH=ooYIAEhM6Vi7bg2?GDuh^LKLi?m*qJT1UHCa(G{)g>uF?&PeARcQG5tsqo=mqdb%fij3)FloAOeH*EFJpm|d1CT!3HU{k`63 z!QZ0S)?^SbXx81FfDt6lC< zw?1<4L5%LrJ-Hrp7KDez<&L8f*>sS(oDI{j;Yf7B^qJ|~p)@(FJpXL&$1-m^hO*O^ znJ3c?nv;{wJ$_)PO+Ot4Y>$<6WJ~NJ#-XEgJN@Ij|JpzZc3!&c+u!Yt<~?49!8d0d zpIu7AO8;U8m$~sxXnSaPu4fQ-HBn%CCdqFi;x9CZ8};6rK4S2?aRu}`|0O?m_)VB+ zHKrrPaN9f}zZ`{b@7EJGJ9Ptp^14@<)PfeW^|u1Jo{pryjXwZ)hOLW}m@*^T`kSaJ zOC>?Cny8`nqvw#iSfVB7hBT2!e3ssTA(B*qXaVadCXFWvdS!s49HSX?QQ-C>$@#fZ zRbO<<6r1Lcn%K?C$Du40d=?OOq#?WjEo2PgR8*d=f)l3{y=amYj&ys#2F%dp zW26fzcXb2NB^ZzRq0k*6(FHTGK$!$zb*c}xVDip`r1qnfZ)1Bhw&$r3!88y%VPR)H z6~1Fas<0iyA;yv(-<|4h-xxEQK`U#!t!5qkg)g~uo_eV|PiuX<1(lGN5+=IWj2{)# z+6-v_vL86Kzg!plG!_-Q=r=o|v=IEDtf(ZoW@)|{Ux(6D0u^Z?DM2Bn1k{J0_&1XH zx1WJU)Rm{K`O#Bb+zg_FK0q zDdJBG@JY{17H?9t34&n)zUrA-;vJbkyaN^WNW}Ht`&q=*r)3it_=7QXnwGdAptbMU zmD+90C`jK2zhKS)evqgMu?j(q0vus>i7I+m`CN0mLpJ2hZ4K>oQX&7c=fOF3(8ojG zp#lqUv?ww?Nwal1%~mmiW~)U!ts97_)CKC*3$DHjO2XFEz+j3MHTv}}{Id9M6Zq{q zqC4WZx^Y91ZY&Lf_FnwHLwoZj;XxsltoJgYxBMNHC6>Dvtaq<_RyN zeR2p!u%W^o*Q2MHc5Ii?6*rC2G^;Jv*f!$_LDJzHG4Uc*J+W1*N?jwiV_zmW{@zUH z#u0oD4zFq8@MnMH5`x_C|U;muprOTlr=9D9h^L73EGz#XMte7$&nL_qI_eQb@}5GbPj9r1T9 zacgK{XV&<<^Wl-l=e>8c#;33D)5qt9=y?sq4`u}-mDRy0tk&&X)wbAQhdzJwFUwbrs zFXZ_E??6zis`LCKv`=^@6^3{eU2#)k(|(yn`*^^#U%|BB8~fp^au@CIrd1e6sHX~l zP-D?PPLTrnlsB-5 zuB`24ZAGdo=Hn0xSV1-AkmK|*vnIC$qxCy*?jOVZXz)cV-JG484Yx zs)=n7&gVcFk3iqxPjf9@(KeAFroY(fu%%`wXNx?#G5ekdqvk$KBtM`Va~~p&kkml@ zt0(3@peJVcn?D8yHL=sBEIPVinHj>sfY@fEv#j}z?ZAXV?IfI_;8nc~e16;5(uvE;7uYDB$T4V+JpVV!-n`%ijq598+2uZyW7ge^XlC!5>|cBRl2j ztYLE@x@AG`f}jZy^r*oMe?W%4(sFCi6NO&IAemib6?D(s|TmuB^_1%M2AQiTaF>VjdU`m|4}Z=~{pPbr_(zCXKtye~-=9*f6>4QKr% z?UC5!c*IhLzdWBix*7f=`HMUWnO$~DyZtK}_;O053gi70qgYDKKc(HIu_@h-AujUM`z8;p;Q#DHMqijT-6F z_y;;LJPE17d(S0mfKI?q8|hbw-p?m5JZ=j+;Yvi)YraJ*LARqcZ17X8kK6cTFrFr2 zJok?Rbny+#u_(ks)5T~7?@G+gU@YX}e+`zMO|K#7!{&6^fu#?k z15mw*L20SIX@3CuAH*SZtSnZ`#hwLkPRm9V-?FX+qvmpO@q7zuiB@_}YIjxF(meJl zm|ZW?6aR<5MT6~o_}SxJN~vRUx)UCy*w}>THNVd#-EKEYy!aOmlQq4$_Gs{ZCH2_; z5AQSXEqH`qpji^y=~m7fsTKI1qN7pCL^ML(qWKP>8wlt`!Jjee=uymX{*Bji{QpMJ ztNeeS0p;$lB#>C*!Ue&EZ^xjF^Azy~$E`TVi6;T?_YeLC?>T10{}TPQyTVtzzqbc% z82)a42O7JKfH3@z;GXC;^AnW+sr$(Y?X}W-+1@gDQM?L2O!j}*O$?2ISFRsCr<*nA+h*m_Q1@bN$?WPyA1RW z)1M=}xPdhnYe56!uhD#$y@LUaxlnYjyG~CYOGT_PP z_u21V_c`QI9;abx4_BDYAbO~YDH#*@+WwYUdXO`-sMLe#EoW5W)UO0nz zre&8u+P!uy7H0Fq+O^$UVj>9EzZrr;8ywv70~ zzH{vOLRlN)Ej#~=XCwWI9E9&Z?2V0LLvFp1gNg1!-)H%8jX42&M+* zVP7mLJ8R!Z_etoZDCquc!#(}cxb4#rFzN~d{;c;*Pm?7*c>pER(@E#r^yJhA94_n} zhvy^xx#>qK0RQr);~gR*8JW^8hvWlL+W!~dk39-}|4+V;AIint%z>1H+=w6YDYbnW zEUpgv$MGvBV9MmI7)28&Hd^Ql`Rf;mzj~^( z`D-@p#B{S@e-3cE{|0eSe>VQv{;4Y;h>-PzIC+|`YT}}tp|SYwQ^j^wsU09769o@H z38gZv3+OKX|3!lDH#U5_@DYRb#ZFQm_hRPPwbP^|J1Vf=ei!|3-SF`v`F1^IU=$U& z|2lC`e>SXijUb+%B#sr#e==aAVE&J|Z3^a(AedhVivfq>y1B+KwnL8pJvMMe}KNu#z*cyohref4Fg2E zwhC&kJ;BNU!BjSZg#xUkkU5`BZt{774zUk|2@mlQ{BDN*&Z-=2ma_i6_YLx5|GyFHtJP^B;KqYFQkM!v60GvGO~V+E*5 zO?fgyBhS7U)wz{wZfuOmsr1HtnBAD>cHil0+vB@r0MyhoVK_gCC49q&m6tN3CKnxd z$1geMAD`q4_$bEpeRKiKFY&!eAf!xrkQVCG-}HKEhNaC0;f)^*-dWV1T%oW(+=*ZC z`jRjSVCnBJb-53)zX(P1(xveA9iyoz`a!(vKo_dtmyQo?eTD8@58>YydMe*X$Ge|L zU3lnDlLDjVX!oELQqo`)`nr|22Tz0&;6^9xSGG78LF+cDggbkT77wCg*cr-3ydh2pF!W&nY*H_yyyTIIfmkV@;hxmo# z{g=Q*N11C;PP*zX_;RtZ7#w^GkNKZfN=j=HKdLaG;%n>DJuqysA%iXXAj+hKA8*}nA?#VyL zdjakoT|Qv`*#u=~IN^jB>w>w9`!8JCcTJZWtS#-XwLAe5PLU(@+}g!S>%CgqXXLra zD>I%;6g=}tuII~|$elVL#PfJA{?SB!pUBOt^q+bAD9@c^h5uAsnR~;Hbpn2yXZ$_K z6F+wxex_o3&7q{%WB5P54TTZ~AL3&E_HQ{J;UA(Wu7wT>9{V3f!G1Igzu8nwI5;nk zH5a~!preAlcuKtmf@AcGI*u3vQP4{-Qd7ftC<-2zH*;}=I+eBlxywhTnF#I$3OHTP z-XE7CKmJ6#RarY4PPVbECa%QKXefR-c;^JMzJ1S%F`YK&^z9kikt$q)aUa^6_84RR z$(wGAx|rP_Z>NimleJ^39V_N8C9yoY(mQzEfaAtPykN$L0g4}VI>Y@{1H~>8q@1O7 zy&BTS+LK(~vojhG$(l`cm>_1c9U*Ud3p-Pr5bri{=fU`)U?O*n($FRPoGSFHitqyv|T_ncKRigu_2&!zv?=d(x-Lo~WW9i#999$@P7^ zKTsNjHHlo0sx%~PMbg!$F6vVsyxdEk#?q&lkWjrwO`vMi*Nct0ShM|gF^Fn*zL=ZK zFY@tXtT8wLA!~^fkyvSA4E#%Y(<36(e4pEcN%J=XWAx1%2Q<;Ijol9M_wd|c?0uSD zSg7_P1|LiO*a2=Wj10EF$tsy!Ga>4=`MoB*^vN#wCl}!pyr=L+bken?2Y2^;3Q=mp z@`@%d3u7@PdTRXkkT-@VKB~ppJwQTGGb1npj$-6kGtvX9npxyx#iQt<=8@to{`uMv#Er3Sc{NG5_lgs$HCj zT4PB~JVVnJo_hi?PH{pDt{%J}b<6%jew@`Y=PYMHfl|o)InJB|CY*Ql1>R+>9}iyP zRK*qA@+HLb=^TS?ys=` z`pN>pzMzD|QAB_?Sy0%Vfjv;h0$>oo5xfI@9dEw=I^$@we7G|HZ)v5Q(jpFXgQv9Eq&>TLpFSp&dy@U9m&W(M@DufW~HWd)sdjgEJ*P^ zWHip>pS6Dv40dU2HNBUH}^^F&jz);yPx9Fp9BU-ME~>Fk|3V zyXe6S7TeL{uY8oo9?@U4IK;bRG*PLwrsQm z5W~<;nq*l18WrJvI54qr)u;~(Aj0tzbZy;a?YCQ{XRiMMFuCi8`b5BJ;yePLs9Hk< zG}U3lVcB*;Q!1#gYvJy{0M2GMVG(_(UqZrQaEnZa&OaD$*X$zwpLBrqDD!;7Sb23( zfo5X?2XMj|g=+)PCo`UpV}9hq@7zrMGTtt;@vAEjHwU5(N5bzk!tbQv_$B0pZ5UF8 zAD;|lW7ejTjnQRfhtv4Q)OU8Af`4K7qP~tssa9C`*!Ld}LL2cO(Vo?WaUY=;ET+)B z^p#YKI8&S-8S#7Q4Nvx?={MQ>5%?>hV|SIT-CqSHZil(5kUWL zeE>zAF*Me+;MQ2v!T*xXa9(l!JpWw6N+6zPmH(4pVx!NR%iw;pcKSd_ZZ`K~jR%5* z_7|5Sc3^7q1Xe;b0$lccz0q8umaJz##riGQ{WSX?=-zI|_c}s@^nvn?=F{ircpjtt zObfBvp2v~dXr*Mm@+i#<*CKniywAReo%%`%z_RRAaCWEw&^NHDjysO_()JVz|3MbE z&>juIsJQIUUdy-whS~U@~~FiDNLMl=kQoQjyM?2N=&^&lwlcSD^>MAQhp( zasn03=%Y{ta2Zc(qjTr9C;6`iaU*t{2zwz20TXt&ed8t4XV@MfrW$0XN?y|Uw652D zC?-BMKY5WN>fzUAw$o?c^atVWLpZ-Jf4Z-tV@EeCJJdk1;=!Tqu1&7sc@ zd;_+UeH!rb8A(qpSf1=a)GRYK#$sk~@{@XRZc-we|qG-&4M5Vndb@-+3?QcXcav>~gEH+Q5b3~^nDnb3GZFOfD8xcWp`Y|(sPtc?1Z z&BL1bVf4qLnI`Sa^9bMiBa-#M_S(N`J_@$Rs1lCrTk~IjE0gXUxxi z3BQ=HCUT9gy`w3Yn-mPnqumGkwRx#?W{DiF>lOG@p%Srdjb0X^9WN09N5u=~*dG#u z>Lcn<_lE;ss31ZMYl+5it@3CXn@ND&L*D4Is?6dATNO(?$HYdF;u-tAp3s}X0Cln5 zJmwnghe*m~1wL7nVB#!hSPN)irPHG%7q1`6XJ9Yenc zH&x;l0+Y&kT+N{--11QJ_u2EonA3#dqeTwE)G3el>4`89v+e*r&Z1gIgfa?*wgei|{Vq2{K`*zX&mr;^9mF2}%f zedEP9R)a(mxphRJ@M25PM?Uq0Ww!%6ePhe3U0MDQ458)8`E33V_&t_kNcza91OG=* z%nh)gBM3joG-GZsRro7xFcY&Uin;mxVj^BlFy>A~n729UB#-Avju5MZvKQ$t2z8>* z?ZJ}zG&}<^f8aYcSD%bIj~zr1d9@AT^YFlq79Bk2T3UdguEG4OzFQ)?_{IacBV`}N z3LP{bhC0c~TrDvttQ$cI78w`AKVQ~`nxp8oVxG}NbT*o}UK#l)Fm`14h?XW&1uVw0 z65AHxjvT`~?XZ?~-!V|I?$w^$c3|#nzxd&X5LT6F2FX<=YdLABwz!K|mwY=(yu0?} z!Z>NMSVR=||8Ve>m3N^%f+_=yW!D2>Xx&AC%)I+|IUZ;T^HPOl0Zgj!s|1qRmnzJa zupRg$4N=5`1gl@4M0`nzl}PD&{eY=;eTXKyk$&TR&DXEBen@^mW%cC{{%BlNhY}I75NR2xh|fF4reFf%-Dhkj-yxf7%#x&WR4` zoXVtwtQo^e=XCbim4Iapf4S^6CJbE!%!yZ9r0vqZ2_Vb9u*t5#B0Im9P{Xw5Z2P}B zg-!_~R}o4|Td6@?&Dh}? zxt4*L-@2CG7GgCHBi7ZU<}hJE7Gn!fZn0LSR$%DBsihslgG=~Ce^9xlUzcH2ba6;W z)H2~J*Z?tp44MFtVRjSXdzcHfD+OYwr+lEJ7VF0(*df1SbrG=*Pc`wZ=?5v(N-rD( z)9BUb!LBqho{aO8=qJs8Yv}Uf|0L;`+Y=6=@nMbGkjksH7m;KKVwoz29Hbo%1W|KD zz@d(0F&0=qB^H7r$B?*f0#sad@bWA1m_MQ#t^OmQiS0NAiNKAZiVitoAF|B<4<5WL zi<`>_gs)aykuCRYjT0TYe?7_lLFkbz_jO;NE%&b^JT8xjXUJp`JIvsBi6Z)`^HTIf zvuy;yX)e+Kp-c30SlM{$@~a=kPM4y;mqh<@u$1K}p@tjn{gxmlNBY<2Ma1TX#?5eF z^=jCzs6+WMz__CtC}C zfz7?uP3$^0su!6zen<%qVV-Gl&!h)A=z5k*dx9_m?A#8|y;SUZMP>APigxd zTA^8s9$}(l6KMqhfavZBY2uPHt+ktElAic>lFG5B61|bQM_z;IGCy|~`k;`O5+W@n zWY%H*Ohb+upn4|RTw`?s$+p@2Z7Fr4K8T9;QGMiWyF|m2kZ#N@>X|tmZ`A;{gVY^^ zG(Azj`O&Ybe=iT^;+;2w&8Lr|a0G5z01{=h| zh&;&mhH%=;FelR6A9p9_DTmGhcahR&b$eT$ne6*g2X9&5SDSlvl4( zucY1blH!Ov2Pj8j+9!mNlse23c!oA=#%wUF*+Pk9s#%?>ej4kl5LyZn7eDSsceEA-Id zFY&{gK$2fME9f+f68@hYOK@XK=^=AA%3@UgzvgF{I5p&di;g8kZ#El#BYi=RkL%O? zKbouBzxB#$FCMYl{h(soEmXo(w%L4RU?jH?GG(~Og4ku4#Dzw4vXR`>^Hp(375i#Q zU>s7dcoYSLHWd}`I~V`7#PWgd>F6lT#9NMU-@iQ1ybJKr-Pky$kmpXUGY;vG5h&*4 zTWGl+mSf!g3knH;;KSISR4aPL4i(EvkMyKRJuR;=Lt$wF zN=a^3z9T~JE_2_nm{&2T-79H}0D;Z_SUwOAn%T>S7>XDc7+PB_+2S{U2*8wJ{2n}r zi0_heNLx#o?UYfHgpj%Mr&w?AJ+Kpjm2qNP$N_T(sBEKh8hT`| z!v|2Mp*?IiKSDMTN-xoZ57C8Yx37+1Y!KTiK3r^SsOnbYKS3$w(Tk7|q<@KDuNqi_ zTo-e~s?ci}chhQ1X`kp;4Lyu5BRBy~>!p3#f%lMx7d4>#(zCS0EK6){=JiVHKekri zy@0A-Pj$gS^q`{+RhyQ)9I5JFT54N#x(|{#(6%u)mRGn};%D>(9ou&OR31f_?6m{N zC`J&hAyeA{za;;t@IQaCv#F=V*150wIq~X#xsQ#Ot-5JrET>c()sI&k%s}H6qVWpz zYUgfIaQb&Bmf=5hK{8MQppfmlm{55TOEYHxC|KD^h+OV8+2|y4$XpJp2m1AE`>}ct zN5l(6^7FxUDv08nkWVNCHdGc3^i&5ijgb+{GVQUMwFZLE(q?djH_(Y%9>BnuA>xfj zxI#CIBAlRP)j>RlaH)ZCsnMcABsRhp50*sccYopah*e~nN_xUdZC&|()@BqLyb_n#>-Z$x_@~;5q=GzSc;XVW+0q?1Js%_JSrkhZ$2p! zjkO*0fc`3I5;Z+eY4*`0skK63T`;)kqx56}J!1Vv+S3TP$#=mB6_v8d!EEiy7;@~d zt}|wb+!C_&?N0QKI?0)NC_{>>oVIiw$l}izIb^XMenafKZmGkrYak|KoE0&?79bIk zpwA=5A88S!*7)OnRGE>W5x~QTokzTEx!KKo zd&r1tUxb%XT!~xunf^C(2w%;3My~euD1JQFg3rtws5ExClZj9q2sLyCvudEY)Ye=lf(YxLZ)EWQz1p1SD6(hPyh1KUjh@$r6N?hr0^=g|);AmU<-^lwv5)mNi&GbOiIS#SZaMBu;pUI%NrIll4~3Tm)F;DeWT+6^JC+4g z8_mQe7%&YnYJXLqeZAKBEfKtm5DXyCd>G4X80S7b5{KhY(ksuhOO$#0k=hfPT;zH) zlLZj{V=CtK9KDB5h71kRx)NH@T*y`Vl=0vxKuu9kO^GMfTg7!Qd7^xe@#7AH@QGrY zRi0SUW4XAPo>$nzomb%xAvu+%)PA5De{3S`kSE^Cw{+qrk^(;}-ia@il0Ou;;EJ3G zow$%|K2w`SJkS6EIUe3Y;8(6*8f)%BVPnmG^rgYem71~UA%3<_K1=dt4fWAjLx|{T z-76oN^0H4}2z-1?OaYe?Tsm4MQ)sQAXNgu#z6$dNg~G%JTj$elZ!7hUnL_JwdO*vB zYg)MEntSBsK3p>R<=|LQhpn98`hQ*jO3~y0i~C2^2#x}WjueF1_u28w>G)Mi25dXK zWavot0GUcyM1PDnMz?kmRho6!zKON;I1wk5xZ|{h;OB{7$R+hu;nKsT{wG8dByE0g z3R@520lHZ{P2SKHlb#?LxeEp_!ebe4H)4M*-?`I2=NvTGS8fNs(%?SE00PcamWG&!>20 znWdd3OmpJi91kp>iuV1es8?20fr`p79Wl=%#bxFkR3!Rp#a4({!WAqN;;XuNfpjy- zCeqVFW*$-;5jW#}OASxwK)=I&bfg(vj95Q)c@ENcBt+mM>C{A} zqzgeyCN@+D&nCPe`R0TVC`cAwVu#%MeKXvUf3HaLyl3`Xvm(mlX_0{93r z90W~!y@<{w(W{FmSj?etnA|ZT4oAca6@?>?Ka}G{uTeDOt=Nnek!}wRGY2EqyZreh z#2?ZwRww=t9x5mPkirj7M|D>8VK>}h<~cht1=Lh;ZPk;EqHhes8{pJ$Evd8C28X}g zKD^SpjZUSgpdyY6vg2K?e3ghtM!wy@qANpP;_hZ(9j?>@Ar&R!`G^pxPGQkVuwtdt zLVL6dWk!hvDN{y*z7bs%EYM&~OZIS+7@IxhKm~p7rCw|{uY3~m0`pLxT|guj;X|_) z&oz+<;WzxO*iwL~eb1NK!smzP6e^HNkRUU^@;xuiM%KR;ruyImIWVAIyBjnCK@b1Y zGv@b@ng^{(wI7B-GnojK8H|_1S>_H7JVA!QaDRu>eCu_Vg8`p8J{iZqVov8aGNUcG zY#BTek6rQrwm0)3J4A{20f&Z^DZy3}n%J@mN97^QXsn|N+0AyibOb(u%^7Woeaa zwFreSPfkz5jUJt$CDsJNEI>q;qY`#Sf;2(%-)KQ;M1C&K#rh65D3ZP0+U-GKVAC(z zh7V5Dk#;=?gN)xN>oUR$MrI5iy4ECpjNfvZmY3b!OD2pTVwD`UhtBzLt8>^SvQ>aH zQA8tiDCyQH-@_J&hNO!#*}xV(`w$!_S^!Y&Y#lVW91{ePVDXI1Z_!1se@L&fm>EVQ zv>~7b_0r?t)8p`TP#RbzwE_kgXC2C}9vY&opPt6bseTH-Lj8zl09&=|mmhTj@k2qX zzKpAo@h#jh=}a?ho!-FWR%0X#E}x#wlQbM$ViE*?&v5PiI=Wf7O9m5;1()H5{%(x z;7KW|6>B}D;zMbc|3UbcVrm8Mb9@DTk;?9g|Jpd z&H%Ap$2vN%UYS#`#yBSG!?4@nRbgX^sgHaq@GQ>^0W=Ks4)Zs)bbW@#ynlw7*HK?Yb&nAXrqzlIONQWk-W+rW{52QwI-=B|Mp7`Pu%w;_*+VSio#9FRd_muv|>TC zYOB)Tj|$)`|6(Cr#?LQQt9q>^i3>vJ^oLO6g(34pa1yc8d<{EP2)4}t?<6s=NZURb zf5pokX)=g!q6me^{ms+26<=jg8dp_=L+dXj&o`UlTP zujP6x%_!jIGBEN(<|cH6x2cmllK`SI-KoMGW%IG|X7&9TpModwB2{=PKZVYP44J_pa6Byp z%?*doiF-q66G2^a30QU&x|9r09Xusd*}*(<_4gXC`tUolzwo4D9jsCd#qtm*79>1f zK|#(EWsZK8=9d=ItBYVEaH=&W7!DxOqY~7Ll<|7-ecYG4k0TgD6bOMo$m_)wh=5pV zEUpcAEQT^r9YR2((uM?i9+B=EJW0V;8ZWtL8;dF<_;^Pd9*%g z;YqRa1sU?L2?$VwVC z8XV#EJFH1?3UXb1QLoyMg-2?VPG}+=BkwyBUeih{@_3O46>Ie!D~enR-a#Il&0pRn z8*L-Z0FyaqNG-Do! zu5Y1%tKu+5v z8wmJ;C!|1$XR$x3!<+;m2Ze_gGecx!HK8vMJ?IQ2tc6f)A}JIye+jR1r)&+_Lx$K+ zM0-?tqF>YmMQET=HW*kpQh^u^h|vi#VM2@w#9YKBz+wU&voHZ>HxA1q5c8X!J($Lr zJZNLKMjo(7V?YbQ2%B|)pr>X&B@Ym*C61u&Z~HyQx^)8~(`J|e4>%zxWmddweLg}9 zeTb0;tO$nzoYlZE13asUaZ3)W#{tG@3z^@XhZ zq%5J@NI%D+C5^D^ER{SPAGE6;=;Nh;BV=Vr_}|5jbe^SfaL);&^H_D%8++9G9t4N` zw_9)HUGoR;sLId|X+1H0{+a)mR1&s5j&ItHr2g~G3s8LjdODw}1u&4N6?%k74?$zodfSR=&4uR@jsfvNW7QJ89rg7jCYSz2YjU-deixxQfUjXn z#{!~Y>r#^Ti{2m=-vlF!O*E?zDkcj4$S*FGFCt>EUKyh>W<)>75GAX$zl8-4T|7^^ z)h4)RBZzyV73uZ?Jf{=YrefDKmR;(=fbT#)$htVwDh6SWB?jpvME&T95NM2xtuY>RE04y&? zb|mh>g9e1cCEAnx>Tdds@Noi8<7l~Kjwc3JA|w`)m-Fc&&jB-82a@#tH?>zczo%}F zcj%0kJ1+7>5u(K3$P-EK!#6#37Xfj$jA@vkDs07PJ#~#TUpXZVC#8|ZMAN3q9_P{yPa`w|+jth*RQ z2Ioh)g$8*r%r8LDX~+o$D^PXI9dkX=V-pwTn)g7|4do;74cOjnULT}Nt;W7h#3EvW z!Rs+Q>=GJEfRP$?&G=i65pAJT1}|9C_{uwS7~}3fd11xPXnzQI*!U0;b?^Z{j~K~j z^KxitSa`Twl0ie0d0K#ol3u-xZ8(Etm^>tGTNP41fRPt(CX86h$D33sln+`Dc8_Q0 z_djhszq2}PJQu$@!gzir-J@q_5rb!)D8OS{Dx0y!^qp7GH&Q&l03(cakC;yv%w$Dl z2$TDreK{T`_m6lskoK_&pA3!0yX|-fm+n4#19!<}@{A!o?-%rdER8-@e1r|hcY^?w z_VtW4e~iA7QxbEY=A3!L7kMeDoi^LSEnHypf>$p9XvA1(@$dEJ#*>-iYADF z>i8G-aR3$w(v`oV7Z&Sm-9TK`y%@{H5-NxBc?ULY&_Q%R#@rG6jJlBW;8h-RmB+XW zid{npa|Q0muYxw@syJq5xN?4Ote~7T)cZ7gJvL$;&}`@c>f|VrzqVS{Kg|y71Wi zIoK3*8Z7T%nFHi1pPoT#bmdjkVxuc#C9$0H6geM{li+Pl72c0|7k@2jT*a^rQrsa3 zy(~4*Ie0xxKlyub1q@W2ND#zZ^Dp%o=_j^iAC~w8S(`^@XI<+v5a5^w`u+ry+k-PiR=;Nb$6CNVCy%R1w}#QjU;{FdI>d7%2{ur5L?JPYHu7a=>xI=mqV2%hfukz;H2G%>eiXf!q|!v(Zm)D%M*55ezL zOb_7ODuLr?nmk*APG;g`8bX#=>m=#b!usQVcBNeP;5Q!y;c+9Z@4?@j?&{}6!O->8 z)%F){BwukiA^CZJdNVymznx8m9MOP{g<~L-e%b`^KSs?ZgU4aCsdD^hkKb-?1ln|P z2#u`d>OxK<`|qMgn)zuSo|11zm%DM~Hn@cL5g}?cYC)XHCRz*Vp!Q;~qd|Ap6tZA~ zmykO5UpU4iNeJz&d(&3+`srTyzcS6z@apO@(xTwNV(Rmnzu0~L3WM_sKfRrvif|Fw z&cEXt!u5Q}^MA+nQC!c*^(kC$#ubMDRokp*FIZ(>hPBtLoK|-L=HCm-X$=S?y5s=n z{c+|+h};N<3H};-&Btc3^%Owr4BA$c+aNSBNI*bt!#Rkm(t#UzfQ+Dy zH&EEgzPjTDo`--z1R>24u@``^A_OK|PYnC{`>7W{DrN*o3rI#_<8!3=h=>3;6%jfr z#J;&BHRVnn&g0959C!&nfwDh7ZRqe@X;1KZAE*~Qlhlxl*HDqPeI#E zY%D0UGAFb8DFBtL2fRT#Df?xRq^OxVQc6IJUtm&(n8-j*YW@)cKSfFg9o$Et&Q1sN zFhNcJqV!6s8(wg6TC}4iKom*>ut=zCsKTHQgO}0@nNb6*A)K80&RY{OF*Mk7U_7=t zmuU}C)GQ#cu%|~SrUxDJ4B-GfJtAJg0Xr|l_(^S@GTl1QPZznxVyntahWfK_MB4W$ zZ4_RBFbEyN4-Gzq^TRH~s;r8WnUr?Qx6i|-uA~RCJF$%g9%L1~GZ|gdAZ{(veV8|Z zz5$yCaDG*)538rxC>w>Mgq=J)_3cCC+tBfR3VLJ4Ux&cEC~$$u#cKXZFv768gWL)w zTIJkO^s73et^b4#8+!cKkS7+<5A4|2F!}6a6jRz0H2cA_^F;4TuQ`1s>7nG1NNrVt zzF?|v3f$S)Iz{54YP!-MBrZFhrIqx4V|>DHJ%_$7(@Og_3KP%QPn#2ptdGiJOscm*m~SKbPDsa|5<-_Edrh>WtKZ^@CD|H{oW z8F3(C@)c_2D{Q1C?fL95#Jok0kv2j%qGMIDzqT?KTy|Thcu6knUxk)y80izVxCxD* zCKSs>fF`-~C!SS3hBPwLcCU>%?#k&hK?NI-MwM+6!&ue=VKPL1H%3-k;h2l0oZ zW6d7sA;5q^G5&jTzK1?nyjr^^R?$hBc`j)ZV&isM8F`xR^MEzT&oW@St zL-7y8OGZB)M(2i3!PyqeCL)25`7i>KV6^gD!n3aq`B_pJ^MYa%hc^!V5ONOtQRpnb z%A14BY7p7D(7^}o?Bh*+*w_bFdIR|T4WY+^Xu-UfKg$Nmc}s2VQBNh?j~+`mYieq%{f+>zgS9A9{jq!dhOY81WNM)#QjQ2*?{Q-6BY z$rfhm4(p}Q;6E@(3@_YM3IBC4^0-NIMLAk?x&{Xy?Py{EyKc_s4mVrFYplT4$y*gUZBf#+Ogf5hSqt3e~o!Oh;e~Z=a@_XgXaZK&Nwgd zO9a*t&I`N%aDkVW^wdx0^8zt-&metHo)=hRe>nicCZyT6AeNy(7hZhaQy&`fyucSB z_@EVxNbj%QmNlO1`m@KA9B}o;|NMD@D`XS)d4cDn0h|}ud=*VrXH22aq|Xcd?B~m8 zuqWQ{q&|kA4iAYux|$n*&IXxGV@L@CGV*uZg2S3W|FXP@FGeEB0Ne*fXxpSg{Ow(&{I>zTEog|LgkASXwL0(r*rYH%q^HLq5pRZwBaDHotS7 zek%uz1L2hE{6~-^WU-R*NK%<3OFWrQxjUkKrEpJt1`&pN3Xul2y0@Oz zIrtViHDTQyZ>~WygEEuq=mpxi8 zeUWs^G5F%}a*&<#f!p=MAkn~pOXT&SzN}P^@;&(zNw=5z-AMFPMD2Sr?s^h5uhEbP zuI$gJQpj~7hg^?^edx+{`5e|Ywm>3a3<>p=h{6u6N$Yo_;Dq_$=BcT_-^3cSD9Drd zae0r(qzaX{BN0D_AKPPA{znEs&g9?O{1^CNj~ps=vHG8xe}(1)NVUs`XheDjIy5ry zVHRV{C(w726OS&9H5JC1W_-;y)^25Ejq{2Qv)8J=O(U9VuYI1M!CoWcBzx^x`2fpC zuuNEEBM77hV^fyA4yPa4BU{6^((=AEK~ zD~QeBgFlDvgXbfUm*d~yNkL?j&4f(j5 zIZ)x?O@{7gt8wvHk!^=uRk_IVp4;ypJ{)?P>B1f1J|p1}-Ur+jxL6^BWOakmQcodR zi3e&v%G^fK!%sz`S?jH4>5lg!)64%L|2%9ApWkotGuQmyAs>t|za@X8IY$0kZ3H%X z4!+Iek}l@w@JCCnItKBwGS{zLt4fPqOP_N5yM(h()2D{<&l|WapUyu&Mc=AYq7dK5 zo5frWl&z{sv3Dg&Ga)-;Xb}O!2hy_CE7;3$7?TgsrBi|>YQzoaBlbt+0E?D-)kyge zTnbiW1vtW|C|Hf7(h3&jD^)m5#7P|`O_hk=g8ng{naoQ1iEzX5u1yx%{yX$1L)A&I zmrB*f+E13H)Epr<&kL!-->e6Rf(zEM_w9*mUDMJRiZo4n%UHuwC2)bU$Fw2Om#uMp z=JFf01Z`FtfSZSu7Q`RY=rkhuu#c7rC_95D{8PtUD`*IUmzs-Y((|w;SB^rr_ak_4 zr*7a+UiVt-2U*BXe&K4T&Nh60Ke&=7u!vwBoRMQBy(>dFxXd zPy^b-6ND+^28!r02qQ*;+rh^k$d?B3^Xt4pU6*7`d~lsxS8X zswx~4g(1McrOoEq5?IGE^fzoQ>g0V(vMnSVI!{ZS`|;MC%=}N9DC!BH2ol#0($v7} z#TTSEZ9VTy;0X7Tk{4SCla`(S8> z%vDr%c-iO7N3c;$D?N;|p=fd1e$6P4pu>~kyLr}ZM`0bAS9D~PAzs!@=E!n1UDjZ% zpv^IIqcIZ2tv5HqJ7JR(&X*=3mnJAgu4c$9_dEJ%_&o$9@k2{vlJtFKes)y;mCgS^ z|Lg8_(SO$bbnAEKw^1<7wOFZh-pjSf0xFSdQUf_@^$-2qG^n}TbK!>jpVj_we57&7 z5VqJTnL)*Sb{6$R&k$YJ+p5cy#u@);oV!yxj* zlE{C7MLHTPXlpYe)n~GU4ndY)Af+ejz{7#(Lcj%LO9 zPF?OViFbj-hrF?UM3*6y!0iSY$+Tg!0iLF+mzR~BNjUI$PfF>_Q5-(fLM;sYKI2o6 zKfFB8nk(r`J)8eC z84j+*vKLXumMz)uht-!&f7$m$Ux|XdH{>AVY7#Qf%W9o}*}e;5C^_t(^)(VSfb!f- zGCW&4ToWffPiQCR^)&#gAx{(5{K@5;$j1T(`(!V4vQwKth9jD|O^QfMR~#wzGF1v* z&?xN!TfJorRx+TNbuUGA-JBQwL|1$vGSiZc8t)G1&;*EHGA-_O-O>QU+zMzec}l$~ zvZhQ38w~=H#M!b(rP%kfi}JR1ed06khL;15J4mgFprldRc@oIj4D|62jHU*=!x9(*jEo+H4<;h z%t`#2$2WVtv+sGY7e;|LE91YT`Uf)oV`Pf}O~4B{3(g%jiL0=f`#F~K(5H8)YnZbY z)Hm)U?7pzwK?>H$*j`HOaZvsJm*iUZOm1=c#s(bZQ95Ab7P%E8O8s_1?Xi04_E|WL zQA>4M=+&z>QS?kc4piD=Hs1-wJS*Wc$dHF~@LB1{ABN8?e0)ml$3;pvb<$L&)^o>e ziHopP0?kWAGB!!xd6BpljB%4}DSN(t*88mEb%)P?qF-xN;Htr1nd2#kPF5-ynuxSQ zj!pKne~@wBYc@P|uCG!fc@fhG@>x=NRrGk34=~33q9AxRj5SQ8MW*TI${s-cll3#W zHPb-7ZvnLxTX|D(mpW-|VN+ps$mOC1d_(tUz(@JTFSZl;rX9~{rI|z)vOB{)`yO`c zsN!!b68pe&hP1kHS8gS))V^P7e;PkjacK~-<0-^(Do6t}*-NxLin9T0&%UmUziI7e z^fs`(t~W>Pz7>H6@f<=GC|+)k^hEIQx=3Ok!sQoW+w-1i1yrnBVtm`oidTs_l<5#y z5ga*3aa~`5Hi^*U{V_kv#t?j)MC8zc=W%-5nMBA)*SbMnnL~%4l`epMNUyO3(hDqu zXOS>2JOP{L20w>iyuxCuY8cG4ie}NC9IqU`6tc}$(zJEyto^5FWx#h9{oLQTR&rYW zA6fg)$X~qoTw0Ikb3SB}2{OJ4>8zD{q6L0|d@JbVzdQGGU(Dz68HEYkA=q>>N;+0q zeHa0igTF#}b$SYA(_=RMXWz5`3QVK~6B9XA2?&{)g%dzfFl8B5$j#@+VW>!5I$Hg3 z=o7u?5WNF%^!wB4UFwUq!k}D6V2tP~1(=)9{TDJ2K#XCR8Rtxg%7~p^T0(T7QuBW3 zA3D&Vp-S>7w4A$V26j;??Y~En!7~fpx*Zh|-l`m>CdWY-jZYX~41RFD)L-InYv8B; z6w6!yh)+M!1=6L?YdZB+t?E;i3;Q4$ zGguda%k~Wpo54w}l7G|wMUYo!1i>Qo=jR_g{efF*kRo47 z!W1o-tKo>pN6)^oPa6|Mbks^{MaU(6IVqXI5GU1Hv^?#gLW&bR)i?KV`T%qoq$XYE z&pRg<=lT$_)+Oe9D_eYAZdL-&xkV&aQYym-CsL^UfL;ZaPHAqE16B5Qj3vf-T}&RGYuTL zy;&6lgG>L&Y-2b-fIHl!l4I3IHcmI%8>inoCwQ)MWVYxF?8U5G~w94OEf%!TGP2myXgKHE)&^@)qA+Px) zW-+*ej76d^Fg4C`oQHIuZJ&^S#-bKHhzFB&cc=lo%sI$4XtSU*YK4#->V;Ynr*cF# zUvQPc+Lq(I71WIPQS#lx1NT$Q+f5;()^&&F@|eP;bxz)^jf>-C$=!#8Ev zqf&mAC$s%=(Ni$&Slc<8KQ0>{@`qHQ3GU1Y{h^Kg%J{VW0(=P9*%%SKBc({9uw9R@ zheb66i)s@Ls$HaM-K574=Pa2`(=y(5a8@FJR|DK2NVnd%^ou%Dj2knGp^APY^)Wr( zl@ozhjy*mTS@G9tG9V&-(D#A%>xd5`ZANiJ)#hlCRUQYhkUpQ&0CgVEV}Av?xIDPo z<1?(k(B?AjQC7a#fAJREqnY%@eI0=HXqz{(qqcd|2lEEjBLMQ~@gx9(eUXv1o$Qa& zv^#i9%dJ5VtpGybBQFpBhbzIgMBH*~ktd2|;3m}#-|fIduWGC55Iy#c5eT4Jv?7sc@8Pz_s?bdi3p}EX=?#s0~PSm*bd{c z2s(4VSIFin=96H0ze|L0-GU~2)kM`&y`lAkq%+m>cG)#%c5}Xep*Fn$X zX$F!431s~D9l{z7 zR;LgG+e;tfRpRVt&|?Us*-)5foaKofEB&-zUQ9!ehej?|WN{VwSE7kqgN%JR-buex>Ry)XNgrsX2ekMn-X%B} z=x|`^$>y#_Xd7u-yUbjzbUk8xlyAI8Fd`-C_K0yER=pGX?BS1yFC)hf;tcieZ`0{I zA@f1{6q|J}NX&R$_UkIx$4K}3nom#67(FX7u~IAD4Yd$vCNfx=PxI2_B{<3NWgsw8 z^~Nnz8$_@9ARgl&D3HaZhSCk?1E3|JIe_e1cnJ^I5ajBNzf=84jAyoT9aq5=-U=Q$ z3H9r~m#OkacmeKOjzWl@{4PEx(CEw^3?p@e{XACyWb9fGXxIBk8?)3(86i`#`WI?m5Zt=Y7ii zERpD%sth83f83{hPwgLAc*ORH$pf6XE;oR`iWD3q&&M<%Q-FM=7#Y3|0s#Cv(_Sb3 zZ5!`+pIv|U{ipL+%)vvn|0;16qW@cn0?0EyBPgcw zks|uDuV~`$k@h~N?QWE;MTT53s)&z#yw6~lVVHRG&1@>GEBWS2dP2kE!+hq;m*}Dy zESMtJBjWieBA(C^k_)Ba9_!&b-5-Ag)3h4ZVPl>+`+dYK(4ie1{5DEQTEd}RJ%!D; z@q?b&X#K$UeUZruQDUBW8qu)tG=Q^__vBcPP#x3ha-WPRep0ySr*NN)!Y|8x(v->G zv+aj${MHI@Equ?%E0fyLeextF_rY3@LLw!REq32}JaE$^#cMbjq+8z(LDNR)#yxiE zhK^$yAP5$|YYBq3;vy=6B$hCR=To*zW3^0fiTg#mAvA&<@i$5k0po4O_W%SXI2+0} zGCnEeR~J8k2e4&oC=F6u@j80_s9LtrXe&(6rq^|lZx;RMG`WZgX@S&L)-Io* zcHG`UE0ZdJ~I>oQE9)$bt!5?S-05sFz_1T}B(w8ePDrL!&~KcsMn}0L z;DR!TQi&N}RrJK1-ODfz&{&uY^C5T8wzi+Stg^WK(tX>G}Nn4rZQCT2#56l9CGEr};+|#??umRdX=kb=nAKbY=a>G3sfIq!pc!~mT@0&j58SW)8! zw78&HMVS$*Ercd0<8zcQE>){mYiqS`tpZZ3VUa{-3Alk+mulM=;({BCw#@(gJ@>vh z?@b1hpugY$$LAB~E%%*!?m6e4d+xdCo=c}&^b(i=iF@EfpgK8!6!O{|Iyb1K3tz3e zpPZlvt-vk{TWGdnKCRS^O=HZPXNoRr`PY9qX3PB0WxR{C&R>A6agtImj8vAAQm=xk zEJMWxPoAUx8He0)NaBc;iClTc8BKA~KDF0}vSNpc5Tk!r1d-O!>F~(vLgCI^ThWg` z5EVqk@tJDL>Rv0hfsEJxhUCcTjqx+#Yr&nPf;*;v|JCBrzjPDZ%C{Dz<^BJj@`VD{ zns?Vd-;w6EgBnu3GEMv9SfC&pn#EIhva&j@f*D`$>MZP#$mj*jTdCkfGN|Bax6%f9 zz+1b?esOaZ{C>#S(ffNU1B+HD_DZc(rezsWC8DU@bsyDi_&WuG)_Vd{uU4`s-C% zOf`^E174Ux15P}|qr1S4*h!f+V4BkF-M-Qjj`$bNgazZ(eBU>}o#yLt7Z`Q3D9*a~ z8xSk1$4gtsT>~b0b=;C49I%e76|vG(OzF4;^to0f`se?Qz8i3ThxHwFq9;B$jC2ax z(ERU*j>jQnnAM;+lxq_zj3bzXc7GAeKDFE9vGBg*g`XOm7B9RQ z`-(gX`&|3u5%gRBW_0!v{XuOl=KlEPxnt>0{6{HTrx-ZPbY!Z}tNb7Hzx5E)F7N}q40&o7ie zO;#h|Fkf3yn2mANPBZq-8;83Xd!)kDVeG$u-iEQqAQ*hd{bZcYU&elyso3vE zly~#`Er-|n7w&`Ckd9X2x8o0!MH%c%l+_ruzfxu;r{qMB;*7Pg%vfFP{TNdM$3O)A zOS`Iq-D0qtz2V5~+MBOx|13qRfKeRj@|G@-oxQIxv)%5AHt^f4&+9);bIx?2zog^4 zul0+k5!fr%d)l_f^zYll7wZ*poA+^Yo?x#j+3fs-?)^W#e$l3Vom{U-rxz8md;Ov9 z`|o(YBb91LXii^JJhglz@kVy2(OZZwLfE#!BYH8d+i2Z>EqZIB3j)%(qY1$l)9^SA ze`oy^v7xeH1eck(8mynsNqy}EuBFq{BhL_x^m`1q7_Zn9)xY%b)p{=WXdpi|cM{NX zqRv+`d3Y-{&z=saNn5@I7S%+ZCQWBFlfC!R6KSDq$HR3qva^`SY$%4<5KgI=(G$)F z$M-;=L)8!AbI?@t9LUtxKI&k;wLw6xZN9Z~XM+0yYGj}B z|5~~}Y;U|p>_=IS{})a#&#>RITJMd6pmFg^77420ig@)Okwwk1zMv!shv~Jm-wkVi zII|lmDV6JVd|rk5lI=RZyy}J;EN5+t{eW&$cuV2<+!>t`Hs@)#X_dK^Zd2TysobV& zDWq9eH<&1lZqw0U8`v}KxJ@U=4ed4!m#?JPbThoB{pdCAUU@@zYxyWx?mF>-8&A^P zBfsgL8}G#{)p9n>X_glMb;O)Ri!!yRiThc@X=zF94Xs$;Y%=ODM-G(k*j0)c2;@L9 zlH;5`tCW8g&+u#z@7SZwx2%JzF>yj59j1$0WCB%a#OI;yPQ-#jhWJQ+6~NZ{Rg7eX z16$#X9ok|6V&#rMXBL$v*#)JtsJw?kn;vP&qT+~x)FD{4+;F-*Z%8LmdjC$p zXZwJ>S|oHiikG0VJl`Xd_~9r_X`69dnw*R{W3vS&GLrLQsdQ2hCodtt9ZcGr<-Fl= zqE@D|uz%;V8&AiBT71dtnRY5Yx;WusuNq~Tf0FZG`F1v3{c;{e82P?26`z0zZ)`j_ zyPAvX=Qah*%9dW`O><8}BG88g9XK9C5F&ql9ZXBOcj#rUpNnfu7(y*q;vy4(YhPgW zIF_4wTTi@+1wPLCYp3%oY&%2Lm;Z)|*vpU%+;tO8t1ZdE@x1so)scZ}vy5He@VZ*A z!6sbS$n|BbvR=Nc93@czVsY6@xy{`o0a*6@MO9}J4piX4ObDtC5E<}#B8M%4S0hd* zz18PP9ux0hbSEAs19dn?CmHxB{zpdeWZ);f04jTVAXN&py*NNUd|89Stl1Te8UIyL zEs+}jsesi z-_0&vg?F-s_?q<~qx;bt7k$iJ+6}RzMZOe>A-1dql;bCwZ$H3UGV{0>#J`AVztdqU ztVkWH*QGSs8cxUHoD>E)eDPQEO$2j-={Tp4qG)xMm3|-tz>=3h1BQn}c}>3$JQ+@C zA6|&go@^ZhMeJ$mr;J%=n;K%CdS;>?c1Y{q*L8tpY&K_pA)_39=h~5#sE=I~S{FlT zz3PnSn*d${3NpB%n7u4$NZ?BP|5{xD)z{fl-!&jb@vU$cDJ;{F!YSLH`;B<~^bf-}yd{aNd`G zs>Nq6{nUV4=p+zSbkw*K=qatBCSyAfhCAvg3aLv+%_dE1TsmsHjjl&WiG-l!Om<|d zjhXPVD?R0}Uk;GeRhfJ8jl@gqE1XQH6;`yDQdZF}R8-;t+$7Qn2frq;VA>vmXAkQL zfkiHaz_mceovt*q^1%uw#x|`X6Pq(=$ZKzcep#foV?SOfo!8`2G+q_RC?nASJs=RF zXN~_=OT7BG(T6-c+ocb)4^#S(=eOcK<9h1D*YUsT!z*~v8GSefLs&*-_$wu{DW&Mq zhgYC->cb{9mC{7zwj0;`Qf~jb*6ENg<+gUNetLk)?J4*X*-t4(P&(96R4N&j+jS`n zMCtGw_IfCKrQD9%q>+Jgt8oE_a-+7-ik?m7c1D_Vt4LRFefLohm8dAUmCdPoc2MEi z3U21LWmIqtsLNJx(mygOxDine1Owxv0MbUunaD7pG@yUcO(;M`cO(J?+2q;4xn&?8 zI_6uu{)Ci$s(c43vMnbZ$0%5lmxp|MjC4!gr`u=PFNU(tbgYekuFd=~%|($8FJ-U7 zPEDVG9yAl3K+iw*3=;FbROPf+9ecOU{Ik!+ZN6^n&q<#TtND^qR|YxU8tspr3@`>J zC9V3iQpNx#!cJDWRkk6oS&&ZOHtlVP9_jd`f8XYPs{L*0_XZX<)99z#hf^>?ZtP=4 zpwjjkD1`aGYE|3%y!=k*fBL<#>n$D!g@{VEj}UV_FTc|6#Xrt3bg?$yt7E2|5~=cV zTm!P}TrHe{)wH_y{oL9;x%Tepb;vI#X0|I|-@^OZFW~|gy!8B3q<#-{j=%nA@>){3 zTv1f5_BbKvtl0cJW6y%26Ly9h*Fs|__5LNMrM;0baUfC(a4}C{c_X~si@26I%0K2a`n=>I z+#M?DPN9hX?&dkLQa8iqEPsPQR&i?x5Zp%wmu8$#@l)W`L`*Q_ftYanp}2meMtWM@ z47JbGvY%1@mC?v;uQru6viNC(uWFUvG#aM=9cD*=>_OY?ufL0|fS#)cV{nXk`M6;8 zpo+EGaw?jj=is(;TwZpppWy3#QR!bcsRou6+OQ`(dR&D%j=_|ZXucnc4KeE8{|{oB ztKtThMpo)pJ7A5(r0LdWO<20a3;_`h!e^tg3lj4XGgO84dML!DZ#6$P}qIZWplTH0*Es`+_TRU`Tu!@vL|n%gS@X{keIq@WBBACBTY$ z*xpPc;q)LCjDcfn44y2u(RvSqbwo`kN=MZTfg=9Oud3=lj2-UOhM8Fl&%9Wq zhhdG@tub7tsH6UEd3Mfs)>|zx8^pn@J7mdF^~i8zqhj?rNOlI<-V979V+pTk7=02- zW5GWAw%&p(OiCLoH}58eBREDf5j)DO&;T9+i^1ovIVIO|~LOji*=BgZk8O?aU8-Uy)E191)9>N>ED8aJ|fEZA< zxN^WmmZ;%>)A4r&FGk^Uh!6}wsF9DS45)TPm`MfWwjJ2O4E|9-iIRk$!J_GdEJ}zn zNu38I`=4?VM27h9AhvMnOz)85SFG7VwiXHe&Vk<^zTnkJM1nLY7(+IYD!E5x!c)oW?0vuGeat927%0dl6^^LjHKNb z=i%d;EbwhM_Tyd!ewx>pm{%5+e;UI!Nq8}<)p37I@FvC8(~1$tj+LX@`G;pk4}KN| z$+G@%Fb>YZ(8c}k)^#|dxBOFo{Xl$HJ0CgVk;$FF)Uk_Yc$#+X;u&0rLfW9U6OjCU zMD6^;vZ6<*Pt>W4UGd4<3GS@##Gxm?T00?_ofg3H(UJRJ;NNjb+5&&$HBj5{N`?e= zIxqt`;uDbY9r;-7Lwp&h(I63$=6HC$xPyfn2rhKq=3zGn&RxW*kFNFk8&yTl`us%H zgOe8T6z|)sRe;WL1UjQ6a_NHt6*`Tm#%a~bT))yCc68ncR~x?k7)Jx5Wn|u zIDY1nlt_vpJ4WKnhjC(Wv-KmWKRBmd!b-^#SuC-5@F^tk)O5+qU7UxJ3~`vy3$q#A zdB|*pquau_uoX5G);{)wGic#kHCq!gViov(}14}?%PiW9)bcvg~-#yy*CZk zF)$rYXlPvpedsDDEm!}^<^r15`42SsP>VfX;uv7PPRYd)pnRYI|YL6(-kN84Yf@0ZUxnxHrWPh#*{PBL*y z&E@z_241@&Ckw=yPMrqzH+YKF^J>NNF3`g`pH#iB7{l8+A;op`m1N)}O=S#{O~&w7 zkbB37=MM#I*M=CO5P{>9o0Oh>6(0QS0)B-ErJtlCi#?GIE@JP5& zUW&#drtdl6M~+{n43NgvVzY@Il3ZOodO%ieS2%ae^FE*phR!((8K2PB`7e;>5RTi} zgNKj2R$BQc_U6gcxzex2mchhM3@}VILW-!IbA({TdnpWiOZLl`h|y&FM*P(wbB$R1 z40`0k6s((3$v^<|vV?j=p2C#(^#tN9QB9`cLEt*@!${;0dfo@Rf2HU?M1Q^)x6lLF z7(uEAPZFxmlq9GCzJ+fwrRtUpyogK0-XpS-Uk0tJ5j5SKtP3HaE@>3~P;8w{91_0! z{|4%K&DV8p;;BIYZs_mpF3%A?%3NxilQ@?_}Wi%W&H2?qsy* z!v(Q!&?O`)R6-hU$q1BZ@GF~EEKQ>p!~_;{i?rsY4avaw0SjO1loPAc=A%L{8r(q~ z>iMT4o{AP>arr)CX*bgM_1VW1vlbb7+<51}=>$KAtu{{~FMO6)~9a~EY zFsO~7$#B<|&tmn+lGE*khL&eg1^|d`5SKW~pt0)7SuZ{U`cySHRy|$0z8QJ9Fk7NT zmsBRk)ihQuzj;*_&s>MhH&`l{PLvEG~vnBZM809DtpUfjqx^Ki=NlXwcpJP=mi zE8JcI^f{rQEx%Q@`~l8)V$BqtWHI*P2s?3tVe@*#^?abSun0Z5#dzUI)j~7hdCN2r>x-b$I3y{r3r`Xj^@8?dxA;~o z8?9Q|+NJh`Y^(rrx<#MUXviR=AK+Fp7Ve?;oZFuiDzUg%37z$a$-wkWal$-Myjdz> z&z=ArX+yK27)Xc-#~ZRozFmuN%`I5LD#SR4ud!e)0}F?sM|LNZtxp;YzaesS{=hzm zBnEOrk1NRs-q&U&e2Y8@; z6dU&`8izapjV&Sctz_U%Sq?zM9bAT>Fc!00FtOKahxEmqBkeo~^aAD4+AT(Ei}ldg zXdrs!&ehq`erz`9X6vnYNKG_*(9YF4(P5R?V(q*dPsvCB_sbr&CaV<7R+yyp-nqI< zw3{^&A7b-8h|K9KG&y|F^ul8EG(8qb>Zkkgs`UPqrr#saI4UWVejf>Dc=h}6J;2MY z-^J;i$S+?`{T@JhAZ_dSA@aGc-$Ny!ejSpdkTuZnyFp*A-~WkAw|>9)2of6S_h9p>;Fi9pYT0Jg^5_i2AS-prC}C(2z>X|{l#Gw?3It=sVK$7syA`}(W2tYYxLC}bU6oHjo_hSGn=M3nqzdlyjgxprZ6ki^%>ZdoZ z5!N^rQfg7do)2O!xC(jLF&WR#mhwGIU&641d^mVypqBG>Ef7xkoD%Rian%xRV-Az# zuyT}hQAmvYgwIEx1e4TPsbw;!g!DgUUnYDv5)y2xcs2rA0WKN%_@bOF_PP8Og~S_u z{M}<%MtTjs)ZXln(%z@w9Z0p5{srFzjbfUxkv{DIcKzMj)c>P4^-pWJ{*_dK4n2kg zg40?({|EqN3krFh3`J^Z^-V7lA)@$~fey-OJTZC?eUIPH5B5Hhu|Upwvl zhP^CU8m8V}2fUy6qw{xwpnJ^K3?!12oG(01FmkWifVwPLqiI{$)X zj5A)HA?4G@t4F!Z4(AD{S|B4A9{mxIoFN_eo6_$0c<$Z#(2|?l_@`uk^-|rQ9&-;e zW~{`t?9tY}N%Q8PH6Sa5k^6G;ebB0SHTI+br5h|c{BF~~xq0e-xc)s;|6Z?uAJo6E z=-)5&@1XALyYc+JWXUC{=Suuti@*8!y8(Z<;O}<)-HX59v+h#+t6I-jy8OjXc~h0Y$0=W(seF}FzFf6; zozvc6hVta^=c-_-L%&^Pfqn*iug^60xAguq>wM`yz2v1>8Fv3UgZ80gNs4`SFAXPj z*$nm->|gdor#}w@To(;9;ImN5r{lBjJi+HgsVRkz%c@TJHfqbug$DZf@t*qc2AuuZ zpPjM(^Hu#7vLT-0R5l3z{7vQ*+2akO8GYYuaWluDXsnt^smVH{oU&OHDdgv z@b`>g-T0+v{Q42T@{V8A4rQ5r&p+JriDm3V9(aS>!1H$k-lc8etv@FnzZZh-;)i%G zMBLG(LvMNRP=lY@&dGZ!A4Z<%+N7C{vJh7fRB`52Jr1-M%u##SmSCfG^ z(Hg8>OgFTX8E79VkWg3`gMKPFAj6|QB;^YfjLwAabe{zPcd%FN_m$0#X zTD(HKKO)_~tmMnTbgT9KW3#dl|I;1$gsmf3yY(bSP%m_QeU;@8bCo|x%HO}=@}D1* zg5R|R@K?FY?*xz5Sq=Mz{|i_7e@XfM!=w<1~DsbeS7aeY3KXB?Y!^3Yyb2a()K<1;UOYf5VAgm^+QTu zY?xh!l|X4|iy!-StFUC`uc+IT8|{w&eKFsSIjfy5;(R%1eqX%Encr1jcV&9_oYjr! zl7OOaQY+>AQmY}spV`#Cgdd6!C3?zay;@BK*`i{QW8Qd1Ll)1I^V=P?X zO+z9vhW3QYq3;*IFpa*y{;w#AHGvv9t#AkfhV&EUEKB^3LoF7OxEB$lYc1fafM*`;@%sGhB6>*K!<|HVpG-m8A{dV6#vyFOfo6 zUc$ccSy*D4!zCuhs57VP7|D^57J^V7LAfQ4vwg(D>|sO#>o?Qs7_lRNi7o97ju&z!ggfeV50Az@ z-iTkFcL|Q$tXy2tH?;Wbez=F!ULC|A=G<48;1BCUn~as)gSnf`RqU%x>{jv9D>rQ~ z-4s73gqU;V*(@;5`egz6zJ>O9u8}ykpmcL(V!}M@KU}j(nB=}ZnQd+76Pzp0l6BG} z;?t@Vr_J*f&;$C`XGc@~r{rSRk0M&)r}T7MBJic(MlW{o=~RX$&{n`ZJ;!nDkR@ z2y8|{y7fzVa2x5Dncr8iJ4G4s<~#K+H`7~iBv*}2l8wEZn(+tYY?Kt^k4R(u)?Hwz zi}Qc`d%^fN!GTvbNQi=6lTp@WoSVgSO4H8%v zGGDbuo|Q~Ky94b;7q*&u0V2;`;R@T9>TX1OCs1Gj#c}$ zpe~f(C4ge)FL@0tVrRj^KT)eglCr56Vx-TOPiQ{{p;;1z8ZBuE&&8A zhQJ}^um2N)pxYW8sj87MJ!s~SkUEwT_hjG&z{OY}^_%(qy#@k|n|7jF9=_2R2+)%_nrIxrFhtqJ`8 zv-c=0f{E-ufO4nUoY%(|8&!57v8Lwd%Q4PN=?&0N`Yb^2kTg?-0n~WxHisGzXyoKT zoIVD^qr6zUelGjmt)JIh-O+~3`q|HtIN`I}dS-;^=hHxfGyJt*Q$auQ zlqhJ#jpznn?eBn&<|yg5NFe2iPdkJf$$y(~>LxUfdQ(dS$9rcz+1&Fi7p<*$cS>Lr zqiz~GRKv!Su`#PLq-?Ckhp}G_RbHh!?~=CGmyW%|g^%tGIt*A>A5P=T>}l1o0-d-c zUEj6+-YQeyW!TJuR!I4v-VYobifPTy&d$kNF$Q>JsJ;u=(7yNK$NC|%Us=C-H=9n8 zu$e!jnG*HiaVP*~4P9as)}r@Ed)LnEn?>#1WHnQj0pBe^03Y>wih*ds^IK}r5hWT8 z=Dh@`xy*dhv*2t{J*dOQk;|d4^)J}@42jhdrc)6o`x6-5=<2B875~!R)*6Q2)a}iQ z*IA4a!xQHS$D_(5$BquAbbjNrNobGP}q*<$?-tw7ip9HQnM=6cnG zQ_)=P%K&KIngWR1&s?{NKWcOTRNDJzJ@?c<6H%+_duSb+?omF#TK8Z0rVc})$PC{C zmI80gz>jf-jAA(amYwc*N6Hwk%9~pky)?{p0fs6aqf2uh{r}hN~1l{S`ir{0Gf3&;8s7)VCsG9Aj4x2PV`< zE6#*IIz319(I>cuSbc8_xV*U;T%O^(Rp95}%%4R6g=Q|l9aERg!GSX6FGu^AWn)ut z-O4Vw!GY*L{t#+^fLH;wMiE?$Yd~!QaO)=u>PQ83EPg6Y9#q?ZVFdwCAz(yXHVi{I z+%X3K)kZ_C=McT0tkPZI=c)SEus$p|u)a|g-bDV7Szj;Khi)sniSfVGe~eEmL1Iao zQnX*_a|l|SKv3(7gOf?5b24}TbSO3k?OI7RqWu@X&)Yfx-Q{93i}_d zw!ZW8kyZkZjIVvMTx`F9N=$9SXymH0tGwP%wFD2bpGx_2aQ?;3;HFxjkQuAF6{?Sc zy4IqZM-fT*Zxzte`bmG z?ff$?ugq6nQLjp=+h5V}Z55_}A;!GP1|Cv~@whIk{1+hYeXbr6CnL!K$+iyyvJ#$O~49);C630xD z9ys!1{78O?w!SpXva%|nJHLf9?PRze;7of^=0F~2+7dT^cEBv4wcMp*7CY5}gs-%kL=Yp!&U3OFn&&EwIJR>^6qu`@B+2Y)&l13eNn%y*J-qYPm7 zYY(6Xg+pFX*#xa!mD_O2rs!iY1W^lQc4A#J^wV+J&(FPQ2Lv$L2zbxB%?6tsIGuTR?$-wg`Vh1&NIvYbro!@0Pl_DbkFt+GwQ(Ub8o3b zpIDLA3{M?rvOvB_Gz#6>Y)+B&;(Ruvwb&Zn3{Pu=Aubc9ML=)Fx&D6}Mw z-kOl7X)80>XaA#UJCmylvmsv9t|ce*7XwWjxzK4^iMX_BESvd5p3lu%A!|5huO_og zk22rWxBIdd48Ru;;=*Cy*{p(lYoM=xffv?~(evuY+D>u;U49u>r=<+*ey^G6?qfZY`tUd7LKmvlEUQ+;q9IQf#mpWw6X zdDcI$Qi64F<`yvuur+bERULr+h_z|tmx^Bn5g0ji5{Mf=*@zD};pu2A-yg>$%k;cc2tq0ZP0=P0!1sNCM)-G5_3M^m&G689zmf05G76;zWI>0My z1Md&}fmhxZ-fqLj?{UIwfuEXdktzJn*$=$d!O-~eTz7j72i_%w_iMn@EmdHxZu^0E zH{qe<7yt^|-Aq_eRHKBJ+)}M5*&mR-np<(%nU90;=N8CJh{R0z<_J>GF0Wi@c%*tQ@hj@{#QEiPyDv{ zpP31MZwG!d{poDbb>8F#|CBD!}oqlTCD| zDG-&ql?B%2aPlv~qC#hUKn%z_tJ*_KQ0|7WrgMG@&2VC1pr^mhU7E@k0zqM)YmBJ3;GZ(=vXmU^M zogL};be?`kD`24C5<*L(-;S@o1Nt={r13rWJHYok1q}GU2Y+cgzM<~|--B&@=d%V6 zzdKLgmjkwZmSn(p$(P>&eJ@p2^t3j7^6l{*=-^vRXle5B-WT5izRUV+c^L8?;Cs0O z#{U0t3g2S8|928!o`jz5>i$+w8`o^tU{saS9!Z!s!$lQT$ zU%tisuYclK=(e;vkoAcj_-`L!>_i$GSQv@(^N@`u_O3F4yo(qq8!JS`VmZWS#t2QkOzIAd z!w>VnA{x{}luQsUUiH%*Da@6=l}%dt2k{fL*UI*0BbvUlKe+C?0-riww<6Fa zTw9Z$!8Ef<(dVhTUiy@Do<64)34P3|kWXa$*kFB(EE=FonYbWg$fTp450H^o2t-4P#-@P0n(0{bdOAs0@P0g$bDXY>l?|yv4?{` zNOmg8NH#INqVA`KS#YPheKGNm_S4KZ@h$E2m$zp-&28IX{A{l7FQvG)#{R|CUw#|& z%EJNlm$8R^SN-K_oMh|iFEd)doBs0Gs~iHg?Ju8OCPCQ@hCK?!?03d6jdn)ALW z!50nl!o1q+F+3uZAm@`9pTp+Kbqi7h{`ZKGHcV#W+FFW5SY)U*)PT>VgHWighB=4> z{`m!j7F`xL`xkKt1w&VCXX*1KeiRd zK$oI$9NHOsC&Eb91p7zwiC-`+wH8K^$uOTm{a=U8hOk+q5IYoCs=>#@9w;AQz+_Lb zDbAONuonhm2}bONKC~CGJnfha$VuL8oeR$3d6QN04_eR+w3F#?+Du#tFf776Do~pt zW6ZY-A6 zcHD!)tHWk@F~$a@@@=Di2tNR|q~K?hnCa3o$Nfj_RVy(OQ5${LY! z!C|Bufsb2Z@`*_p8=O90z;L|^n;xV!2p8=+83k!E4yX_l(ibr|wdO~VOqsia7Tt%z zRbE73S;o3dj!y!@xkgz{UaV^*UY*z4J6tv(=9?Vv(~6g8z3t$~FBY*5(j z5jHnVC4S-WCa3-3IIQb_M%j4Fz!zKQnM%lS&13zZY{!{Y|wgZeqmz5k+ zcoH{|`1jO8|I+Hh#YW-cGYf04FU2Y%`*zZaoF}S=>3Zj{|80XqxQ#z*|grqfl$C$ozkW6V*Polb=Z6b zXh6T@vMpy6VBxX?dzW5WJg$&vL7YkTO>0Ai*%o#ZWVO5u$b$aa3jOXH`wG;K(iOsz z*nZIOr_eVFq&Wqx@PJB;OxWyW;~&G!WD=wi$RCc5NZexsi6#g!*=(MSDNLk1l;>Y` zJ9I%S_6w;Z{oD;|#bgg4+qNgiZq8*WQK1?E9z)Y1+7~Gz4Za}O zQvj-Gsv>o*K+FHcudV@sZ2F(NPo?a`_$a2q!M4A_^o%cPNjm?urs${h6L|D_T4B`L z=6i2{dfEExR!@J*AYa|tf9RCYJl@-uhic$H{6W*%t!Cl~L*$`FhHBYra(tw`YwYq! z`Gy!)faimg@bIh^57Dvi@%~KkXM^=Lln(4c?wIK=7@OaSRroFs0F2Lus_%^ri}L@~ z1>wdCuxhBspu?dZ4{q5=yWW!!wpc8}(cay+s6Z6Tk~ghgRQ0EXj3c?u4#S1J_l-E(`DY-!8D|1=SSu7tsY z&192JLz`$vF@8^x;F2X6FO+@8OT8hv@Gy z^ju<(+M)z$dB-qctep?T3&g}x(hv*=<$_iak`2~}Z;)K(7a}Amiq+?t_y|A6>ZZVG zoG3y9^cdC6#;PAWS6}-yK=F#K*nihu_@tj zKFZdYoQskIc6T3jlSvlwR_)2K0K$a zRdeMyq-LJJ(1ng`#o*}$*m3wZw_d_&{9F-S64)4uxir>`LQAv7fiO8@eh@L~8a`ee z{6XdR&T_ST1g@>Qm}NyI7Dp`_6sSdm?l^gk?t|7p)06N>*nB%;B4`WetoiG~Vhj&C zk@8@yB(xYoXL6jvWOE%3+7jatQzq+E5o?$g{))8|^I-Egw)_*{Rp1C6Ol9Fiz}|<} z^XN*Tx{RI_Mxc?>oIrVM(q8a+RoOktj* zS2u>K#^=H%Z+ke7!?ktYy}I$zI1Gp3@~SMZ?@TWL>Y98E6vQ2EZEPLkX(+l8mIh!Z zv!y<1DXgPzc1sy+WzY+~9m#bd)z#u2xv=5_d*{3A-|781{k~oKNt3_!`-}JdOXXjX znKuq38mBx7*mll299U98OHjtLqtUZ)bXvr0j&SJSM6O^VqbUp(j&V(#3Y&yCZ;M7g z0x#%VaHeK`H9wndW<|dUr+Xh{zdXjqRDW+v5Bfv?@8H{pKWe=e@ugIng<7dDzZhFi z@DGbUTQM`-Au4u{QC5+64MV0+hMPYb_$_v>L4Qpts+m8F39P!11u~Ud4nK zbMzhb7^KF_NqLb3{y4Ws8ei(u@+SWx_ESzA zOPsRH2?gqf&%jTEf3o>9csx0t(^`zz6q(*I`BP8((&|s$=7O~qobuxoR>L$T7lOWo zokeuebcGehe5n=UrROoKT0M_33R-pnXTAkC&O8Q>-(yoakHPh8xMl-mLpg)bRB?4) z3zj$l2eYJR9w1NG5e3+Y!)a(c9XO$4`-DBf1M?}zWr9{?9>rVbaxlb61+w%|UW2v` z6}q+|Z+b)F{R)Zah%JOq25o8uuQ^+TTZ0}cc3kkgegK9Kas}EJV=W_=G4zH=U(A;$ z$A^f!#U6I)e~f2CMv>YBFucZxjIaBD-=@DVSCW58S8Xq)-=|fPUZs&%9uD6YnvY6* zwr`)?{>!(larlZxCbsy34E@y9il5cpyrrNx(+zPGN; z#X2|+-P^6sN6>W*(RD3D$3a~)W2&x!s0(K+N?pCNTh=)wA=#{j?h0kiQbt)r(0tMs2WsTY`~tKE z!;yPpqutWX?bGn<)8TKx1GeW<1%EbU6*>uD$)o8j7T8-g;xP@lw*q{X27E9Oe?>D2 zqx~x)rAjdDo?yeJzF8#33!V=@LtCq8-AsN_vxHY!yDy+ zw@AYqqTtOXF+j*p;Aas9r@bRJybYVAy&KQ$G`u?j&jI%@+_1R|6u3eS?jW#Zon}Xm zjNE{Jlm0#l05mJ^(y&SutjH9F+PCZP+Ytl;(**sUyWVw8CAy}UnuQ22w)_?{ti@r##TxWp_^}|rCo`s_ zbMn!|PaJpw4exOU??whqbPnD;54;y~$EHtJ@cw;zr{NWQ;Kek&-U{A*-|sLy%`azv zly4*224|gew`j>YNP~Z3gOKy#>J<3rZ1rLvi?zWK3|b!h)sGjEjBfq=I3AEJ6IDGW zx*46;zjR{ACi|f2OCo+Fy*(3jJ@2kp?O_OmX6LtQN;%?77c)<3|RrSo%^>n5^ zwtSBg*tULNqyZ06fH$6&YCs3|y+vFc`5vj^ZCED+Ij7_BM1J&m_bY+v8c*NB18Khk zTcG9UfZFfDcaa7>L;*hX^bWV*jqis)O5@kD8f@b$LWVrPhg@_9UpM^vbod+afb_Uj z!QUPEcJPbR;a{oYcT?~e9{_w?-kPo;ZESf9XuywSRTld6j#&!vPK>wnJn&w`9SJd2 z!P~``ozthq9(XYgueXBtW~9^bwy;S~dxvRw&%Y$?y>wEi;Vt&SdmVRd@0AMP9UaHd zEl(#3OjrM!ufZOyz%J)e2%VHyo8MdbZL0km@bgX5{wG**$N1yMw_aen@ZEq1BL52P zl1T@SZ?Om8mvGiqJk_Iyr z%$wN`!0fO+C-&jdDR=};c>OdyjblgQIsE5$DgJAq&##q+Pt-s=(cc$);JuDJk$(lR zF{~Qh33#JC@D^!!LlnGsFYPou3!ga6JxPPx_@Xp-o}s{X++M#6&@_Ly0m>tfw`kZ$D%hhm?4hptDT1asKUMQ^T3?**(GyhK(AF5kl-yn4YnPOB46%n$`?IT5TfTLDZu@K+Dy>MfH-C?3q~Z5ey-jeB+Um!#&ng)*h+LK(u{YcJ zJGNh((cf{buD5Zuw0B3Ps`p#_JG6ZiMXerwK92`%=Q*mr5UTIAexd(a!|$ICf2xMR z3tl?#Z>FYWr|p?G^n2u@jP&~ikJ+9ZReiy#?}&ch`b&@l92KegtH)n;rLMml>W`R# z9piFwhV$m=;aH@lzbaDR96cCIP6%|T&kEZLf-2O$BDtA;B$Q!QTj}o=IDta`B_4nl|cmz$FovGm6JpMr74NSo!Xu``?@V=}#5O^>D&}lD06W(7Mq`kkw zo6fY?9$)6+J4fHH#SLLjQ!w8fr(kwO-`ens-SDo}@catiydQKNo}>RlrtdX==NbPs z++{CF+n)?6xJSA4?O=Zaww*J7g1PZ?f5E8*lFLmp$Cs#?FoV`oNyOdtU6fwm<9I;2 zOjPydReT5aiM{akbcdglbR8c&FKrs3+mva1D&>%o=0O(xz5Q{z>AMT?!zQA$mEWL6 z_my|>nC+RX>MCS=Ve3?aezyO36zC_%Y)8xuLO&8SBt$gK_5Olxij;VDx{x*`_tlED zv)N(i8RpYPwOMFQ-J0oMXR~{J1Ao7qmd4+)8oxLHCbS4@T6`z`h3la{y}k{2KzdxN z>RbLjMUPJFW3BIt(&1mJ;dfK;v$Xtl8ormmXvvDH8TIX6JZ5|Dcuw^X;sgGA#!IDZ zm7aB6L!DY5fP9@I^|z6)`MUmtRsGiJ|4;Jer5EtM>B5ZtVFMlu|B)XJ;d$jl>c!7DzWwAXId|=#WSKmP1RG(wxQn;eaF5%0Do%#^!7~E_3V;; z%)Wy2zJvBW0U+t^*@s?HWZnaVK|+3zFwJ~CTr0Ow45h`c77kA)Sh%~J=MbUYYqMc1%BHo z)yNS5|AGb|v>OSlRt=)&25$>#g2)LR8OHBwtpC|M&3I`UoQSDb{k)e? z1$LSS`|ro4*-sv)z;@cc)$`BaaXr_qAV0walHo=LzwleauS|!3i-v!sg8!Z-OQ-M` z8*V!mjP;mviiY{>V?vWdG|Z0rLu`4y6A&G7{TFUXhXo4e{UwUd9oCPUzLkXemJ^+} zL_IvB&xT0Q*qF<))R_wSaE4^Q*JFVXN{`jb%NuHzKeor7=FV=-W-`tR$w zBSEfIp#2*4ea+WOQzMWKZr*y9eT|E z@yh`3VWQx``wDk#?=1@6Rl_?Cuf_xKAq{V=f;WxC&=cf#^u1fYy9i9T{?lOp{YPo` z;t>jLXY_~7?}tzIv>$iUeg(Lj2D~qMbkWm|?-YUQ!uMVc_GAV2;iEc_uN(fClZdkm z{#XtF&4+~^OO8H3_+I-S_S{XVDZ~2A20SL2E>-o-)62jfd#;oILYp6j(l*=Ry;xVY z_YcyriS4X6XV5oI{KS#(fQI+Df_G!FBGkV4>w@Q|e^g+m6#5IeCpl^q)G{_9-QI`e zV80sO4ZnXH{An8g?%xX)`W$rt@NN1o$C|i9j;OB(#sHGJzKp+{Jw?U`S-M{k=R4`c1!p~p_#5b;|T;CaQJpoiEa=K+AW zM;_6z#w%F;^_p#G?U6iyNy+b78t5m#la}9iq(X3C^6S!1c6*l-52wLC4ewqB?^9OR zG5u`AleK6E-ZOws0-mPe-O+J)9{Cp>UHWaR2D|G)X?Bxt_P2^(5Pm&^m2J(huW?D# zf2r#Duu_pC6Td*#wAHkX^RJNUn*^X4^ye+Qt|L`l_ZO+U(*2!GB^XS(0+~$C>kU-S znvDKVp{G}$1D_RWRR(2AolU#Ee0p-;lX^`NWdQ`PC+^iIiXgbLGe5Bd9~b6$28l{ z%!n;=vyj$?TX5w#?o9J*;D7n}jP!gQk4evozZLxNKkWYu{;BnhC&m$VZ+!SpJR{z> zg0PsLhj44Fg9BR6@XFtG`2VHSw(Ebo-h)-Wc}1$;w*Eq}?tgN9*^4judlOb(GxV1Y zcuYE7D*LYk^Uz=ZPsk_do9N}QasAtE9}dy@F2j~-@O7SEum0BS<6e59=JQZ<2729# z$E4TEs(xeW|B8O8hW`sK&45pDjn5x`BmMJ9;6wf)W2+NiG#aV`I%KidFj(n8Oo$3L zDi+6_T6NGWxx}Sa+Ffd(yDpPHDKUKmfv@skGM7V(9-5}%Dap6{9zpjgU{7?91-j3` zVZd;x!{mH5_)taRi8_P_QSy%pY8t6fhR7&EXCL_7G}fk{8=sAMN_=K2d|EX5id^^< z5})aM4nU17sO$%e4C58((IaF1r|bHyyM-PraZ7r9*Y!hwZW?3r!xNvj0gt8sD+GcX zfsjjnPPEsh+%OFodjQ;?TVZ6N;k4d+aRW+Co)d zXZ(Xm&x7<3!E+M3#+1xX;~q~ zr=U24?*m^PoSW5xZ5T|De4o53AzpPi*@J%>ozOEq$wD<_45W)+-L3W~Id=gv#YzbiD z+|@>6AxD@W4{R#q7<*>sB?CCKCo4V+bOqq}tRVjISu)n$hz~v!wKJLNH1LEtp#50Z zO$O?aWfdv8BBG?z=&np1`9E}~&f{EFibN2fRUio;@aqX)fkPFHu`zv1xK0d~hW@5` zbP7^B95WwJV*%9nW2tWt>Wd$9F|Jz&nZ6703ug9Nk_;S;7p5;F4}?EqDZgzsA0MwC zPX&KIKS)U@nh>}|^9QC({y-`IG>U^FwA(^9YDhQnH-41J-+3aNyhO^dK*IN*gOS_X zZi01EP;m@v^D?dIW zqpEp~S|ombL1p}U1ZiC##Gk|H%hARZqhX`xXGyE?TE!aKYL%c-nC5WU)^|-@!ga>u z0KTLmdi>|27;e<-yGH=1y&f@;gb5(YLSrtuPFf<;YuE)W%4{bMjhT7a@F=FA#Bv>X=vYR77`xW^#r}LnyybSm*QPe zT=Yp#6l+0PC&l!1dDYU8jdPS+;1tfXelu7JW}F4g!MC7*)*`3a`RZ_jD@Zp~>5?V; zP$p1tl+z%UiOTtx3~Uv_5IH@DU5DcN7(_ch3j_sJQ7BL*T9Ncc-6CH>VEPcSP6oaQ zOes~OSfE4PwAK>V5&cn3N`Wb8N9=nJQ^w+{Fr`fD+J+bLF~e|8{V^E7C{6vb9WQVW ziaZd0beHmX%j3_m@(Svl3x2FBKrOgV*B@`8NHP#banT<@!`2HB=LA1OJ_spOEoD%^ zrTwgLa6}uGL6hRpM<}Va268rr0JCRGaU2H#EQ|RdUN{WdCb@Q!0YhX$81hrmrz&X< z^Z*q|GEgWC6Wvj_GTWm(^5xTv%40KLY2`6o`T~?j9ux>hY^6kO1|JX99R{z#YIrg* z6{9Y>Yh=_NWU-9rXPvF`fbmFmnE+QH`zD5o0uV}kuT->Caf-C*y9jV2symj>RvE_Qm-#fv@ojd4S}J@OyxK@u)n`R*xrv z-;a=W;P=t4@tjko?Z>iZ^(d}`^aHgh*%AOcc)L#CUV^vig4uXx4IHQ#{Gnp-Ak;u% z4Ta6URDpAt5SH~!tt`7f7^{hxFJer0@{eWik1>qi;|nS8K4a0&) z11Fvh0T;-F$fL+hm0_-r#7mNakMJxKpQCy(pdt+pjFva>v^7`KM~~M7#~grC7D5IM4Yx>D>{)pB+31WOj0fOh4>b9zNQo@^-?@P5j_8)wh`dg&fhtL4O)WYvWId#BR9EENnwY6{un<@6#xLaWLEmU$)ML{tK_d~5=^Gk}rsk)P&Tz7pio2Q6>Qt-v}bpb@_ zo*@2sQ866gKR;TF*~hp@25v=#@iAL*-8$6tZNe`~2PA)EGH@4On!e}cp~%{^^3AF8 z__yjY66N_`?x|!A(TUD{;ZU^cCMjNo;+3TK8_3d}4EzR#Tl`AJTqXr_Q2rX-w>U^on+_q45g1{vkS;0t?o$bJm42~@mk+D#2x_fldh4Nhs;y?aaM7P?hWr;* zLpAd`R;WeIe2fe5e;L)xR$L4J@$2S4UV{JfQ1M^BIaMD2Rz1F$_Jm$tEpzc7vXFn1 z6fZ*YDrWP?mTgDfU=S2Mp>XQ?bt3;lDZC4^D;>bw8tcA(N-oAJm2*D|r>mUGvN=Jm zay9`9+ObcS^OU^z*MFH=<-9B$Q7UJXN9CXe9I~e;1GhtHW%goG(|9Zp6Q4u%Xl-*A zw9R^3+f?JL2-sUA+NRvrHkB!DGuDQ?yA9k@LCaP;T8k;Avs9H|a@$somLA)lQS%8(Q80Cm5r(%a^Rp^OEN_>IFb6q*{R3pp zqRsyLUyJ%#gbHl+lbT=5!BY{xE2aJd)E^&n9RZIfUljo9wn^;~3_*8epFvtxI z(&s8J;A!dWhLJ*-;Oo+c)@EyPA0>gDj9}0^7O=zSe>mt(vgaSUWO<2EzBUH)QaLsb zQ$~>Fz0d62fI&|4w}j@*bI*(~<|;K+$%^kIP=2QgN0#pQ@_#<0^AcmHD2{ zS30Bo^_X!hzG7el-5)Dbe5G0$>kKP**YH9qBN`*1JAJr zF89LGOL2+u8AscgzSGs?Y2av$PjPe{I66gh^dz)rloa2L;#KCxDszp^(PEYdN8wvw zo`1zhWMGDchOctut9$SjIJw0-)2}$WMRD>XybU}4DB03xPqf2B*(XEr73VClB<8^3 zF$;X3Q3x3T?`MJcv%vdV4)1@BWo3u=y>OEZ{QfD$`*$QYEQq5O>$D0*9%Wm2`#a_^ zwnWgrkQBzszCiC@C*lCP!2B3YyZ1094k3>?+>ECp4maW=88`|^#K+9RHN{~Tew8?s z;koHMU+zU5YNYt~C;2#}9@CgTBU_0>1;pVjEe<2W*rTNQY7~co5$1v+dUp{Er}b{# zv8PfRzC4sD-GYxG4R2Ug`AQnzRnjm4Aopi+v}ZrWPN409(?1KK43JrTQA6tx{1+Xx zZ};ZR3>L>kNSn4e!nVaRh}{_O2(dbFN;!IG6iT5B9`DcOh@!oY$+0WQ#{c{YB@{S7 zfC}MN{K)*}Y$|y=qeaEGxmasWAQ)si*`M z#m8KSYwGuF@Qc!pem@0oP2UCbRM>y6eAN0lA6KZyv@gcxDE6n@7e`C+H7G9j1(mx} z0y4>F@24NG3%^d}{}*e{$-o^bEM-#m1xl^9zSmPpfu7QY09k8y_>07EF-q(Ua2LAZ zYCu6lhRKx1JR36xnbvG>%?G8Ty_11zDd4Z)=_m_WfZJkEYkTrC2%oJikhj$}r-8E2 z2F6I~xKI~u4UD7l`92JcPvw=@z%WlEg3)oJl9GY9rE&mnYhi2>xeyDZzZ_tz=1MZJ zxlJJN)5K7NPacTu$eKBX6d(4Ornt+*Aj!Q9;CMvY`a5LnEJz{OX>cml?uhvhL^X)~ z@oZ)EB!B&rB7egGol_!_KUm%3zZ~2TPeuNEOXW}g3Gz4i>%mI?KF63dtA ziu`TGWil{J9mzuUn3(({4iLp1 z0?Ki>SMH)y6BBpU?auSxSd20__*vxBFbk7`wPYLH+*?pAj?aVMnTfgVOjsW^uq|g| zB-T4%qQxemhabS^(q_09O>+c9~sZj^VY*F@qGreh!d9 za6|lnv)~h&HHu>@##fH5L3_4dg%|^h_S8|}RYAD!BOK3n85Ac6=OAtRV(OLYYA0+W z|CT`%rQ%0et#TMCQAl}==qbYbC(40Q%S8@?~wnB)O9jHT#pnyf+oB# z<_WxCecd^D_Ikx~tRXwgAzd}h`xML?y}rDU_-`6w_VS0bUuLEo-csBU$I}(O290B9 z;1v=F2f>Rqz`b*&(aCRAqdOQsVf_JpL^Mre|!!uU;m-a7=B)SyV!+iK9KB;Jg!bDB+T&zb6ycb?`!yD{^mx7j~0B-aE zjGiLjFYtVSu+^R-h){J2d%(2Su>XdlM{^uX zoRMwhHXHH4$GBnTx=1qc^h07)^hLo<-gjeO0n}&#t3wrCSjDZVLcSJZl}!8KP6fu3 zmJ_uYr2K_4e?P}q_)$nk2j4%wgwU3*G2-<_C3u46>_q+$d}@Ab?cy|n5>bX1Dw;h0 zbWye3AaDl?(|Y>9D3TbNIB_p@ALlIo`Uip1k_ZB7il5z!%Q6I^%h}KV2(EC#vVn-} z5`^|81E)U>g}Cq=3ZB32aD`4e!VPfV&LJf{*AAp|*7KU>yJR3oD1&t5N$_*Ekqa`^ zuigDY7YH$nOii@U#DrSBYLKa{Ojo`pqwbabKOKDuZoZrlm2NaJv?}4d) zuY|_Y%t~nh=>Y*CT{kK#3JU~8HVP*vLO8VqXI;d5oSNuWiy}Mwv&iTs^oTA^v289T zWb4bDQ35RtIX2k>JP@1gI$R_JhXA4Y;7f4L^{lC?w5(?xjW>vYkSAh~h2?{7@GO~u zhj3lGZPDh~K~`TyMDtbuZL8{5<-`MVysKOFUx@L+y!txZfBWiW;8A{;I*C!~+wJ{` zoxgq~h++EPle<`$eI@81ya7)I;g@g$dk}EqgP*}Q*IOS`pK!hPI#%GjM;-uQbmkyX z_?3KgguMHSdUqz`F1z|jm2fgJ0&fg6|2Dje`(DLk8s`YF3B0nA{Cx4Zil5y9>L`M0 zwtk3C35{GC&mVyY`1uXLO5`7nJB|b}FJItSxODMD^gX-2%7<>4W6ooFEN&d5&u2nz znrT7A@lZm{(=nB7kt?iuyeAjmaL$6kV$1-1hy~5~$U+SP9Y{U_fZ}A}&k6wiI}h6c z?n?oH+jq>4zNk4qaua`mKuE0zvwnEe=Qy8Vrmld=G<5|-^enjoP7%2foZ$Bv^Oyp8 zy9Rmm5;g$$8LyVI0FW<+(1r5lUsQrSSuvLh z1lG4uHtw5=x7IdH0H6-75AbVRKxP2!=UD$n-vmIS^04-NlT5Zit3bGXFTxRx_fV#W zzp^PASOh>Y^50Qw$^QC9qP&|LeR=p=#-#J{H5Ad@E89`DABVF6Juxo7=j8cU@d08K``bNKNJ`aamp`qZWCSnM%lm^0?QAGJ3AP`1H`KSIxDp18IaT9Uh8#o~k>hr1{_=)=pdlN5Ti@IC&vAi?Y-Jkel z`i6ZbC7)L%-&OAxO9Olh2`uhA3_q>I5F8K(T2`QVKgM?;L0^Ew(dyBWID#o<|2dQ>%Z_&6 zH7-b^%QyyV+#RkP-H;Vd^v8It%;yRuxvGH69Xg4Z7-G<-Ke93L{0H!}dflot`!xOD z9#2N2G=w!Zo*ahy8@1^_f!DTXF%H#Lt zu}r*<$2T;*U!H}YI+*o=BI~8>r7W8oL)1vA{5gkkqu7@AFyF%d>1x0Z|B7Q~VI3{Y|BD9y(uUQIzVmp8 ze`*ln`&0FMK56uw;(3oHr>v|SvY*V>V~)Ppck9;S?9uuL|7|p_{f_lX;{~ven0v}gZ?)~RBP1+{R6dDLnWS~I;Ef5SyYFe3Tl6Fr!p{WQI zQHY|osIL^7Kr5xBq_^#Mw0h8k7Zfi?@#s0iQMoCSv?WO|w55m^>I((+utVU`g3y*r z=l%ZH-ZQgj5TgqlwZ*OHS;0J#y2L&c@YyR#c=1UtYhu_ojoTdxoFH9}@5n|AA^y zdj~?}RGR-h2pE2B{Ey-FFB^S#p>HyQepv-C0F;iJV3tX*X5fuY#${CgcW{ zETv=l*AZ_2Jn%@vdN>MHGGo{HOnImUXw(-f zrF_r9;;y6Izw&{*^Hu+=Vayr{!ZwHV|0POG|!gq(hKU`J> zI;`FdJly%BmZ_?j*jmCr1ep1zZf)Q_#zTYQ%tyNVJeCM{o9^i~XLp?{={!~aD-}pp zo|vvaPA^M&d16u5ICt9=YCcb+6z01Rd|I093G3Jab5Hu{;j~-HMv_~8?csC)$#&Nk zBH64rL9*$q_P_zJG!0dMQa|BVl#!~dxO8u!o!FXlcRmL$F%OLcp_6Gt|7kkR7DF+W zxRFl_DPMU*Ncju*KTbDD`7E~}rOS`|m4(3O>zWGlpXP(*eRj`t#huZf*GjrBa;nt7 zboI6WC?E{FA8AjgIdfIh;U0lZ=wGA3cTIGves0Rn^2?q{Sy$%)0{c_w>Z;z(kUKW_ z(Y9^-tZ*M5IR0tH;KP9@@*Wl5hpWbNIAV?;aLjdb@`ioM@Lk1SFN5Om_Pyoow0gFb z(B@qex?b8i;`_q)bk#brTU{)+rqJuRZO@vV9ajdvh2`_>w|WKxXAA3_dqsRqbKd%O zR{n_x8EOW+Secsi%&~@r{L8EtZ@oK@n~hS-DYbsef=YLlYt&7h$MWQklCID`t7km& z$)hN2ZYWYzG|X0pd`N@K8~%v|QehqFQw=QsTu;0aS~jy8IK@%rz1AYQub80#=(6zL z8#caAACC%;XAb-GaYspKdBd)u@|lOzL(lM!PY>InqCfl#(ud3koNO(1lrfZc)n>uP z9dVHZy@(@efv>b-m&Wx#91R$?JSMnAO`w=LQ#V}K!mYD)VfZEmkm(R)ttR*Xppp;& zFnhM{m#<(AkRvj~%_L6%4y%r-?aSUs4f_eYbV0(gu)1AXnwLNsq_p%3M zbB1Xi$~f_fdG{W_Yt^Br%xR*ch4rS)p6->@(SmX398Rd5(mau4YcY;DPqZ(L2tl(G z!fB^1cA!qLK6+VNFy0!C_nD6sj#nepP#I+^^b1^d8Fg^Hh3l*E-i-J9?`FrFZ6E)f zJb0V-KQW?xTD{b3)yEeF^jrwm#DtXfkWM80#~U_VHW3i1&;z~dA;|&eM(hh~?75TF z|K=!vxLMA{`66^fIn;qg5>2M6e!=zqYPcmebJ+bWlz+W9+p#SCVrI`vZADF6ER&cC zQNj_Zo+_s1Rq?N%QWdGtVJM4bm%qBPxbuknm(l?CD|hurP5s!HsDnwWmylrKVHTKI zKeOyr{OFiq-!Aei{wGs$slN8QNMm2svw=9fSQ=U0>}2Gn`!q;GtK5e#I-C};$gGg6 z+G{Ffayj8RgEMBl=G61-co$34TMCJZ?G2KcLkg?XF(v>r*3Td3j$evv5+pLUxH#j_ z2OSDh*M?E4>rcjsy2@0^VcxL{XPwpya412ENg!1KGH`I({7C1xBr0Q1yrN=wysU}! z?$k<*oI13b4X<6D=Y^}rae{8ZpU}vUPaFC4($9W881g(D|~N z!@?i-8ot(14ZMvS0_sps|DBq_u!&MMD z%=s~zMt=6jKVIYC>W_X?lNtSO0D;lNRKU=5L}pe--(j%x@{bK2zA(FRJ#y=fKR}@j z-v9>e4qudUQ0ZAV6)WsD_*?W@EZ4;=#>aa;SY)K&&eFIuhR#m{X*xv6I@~RcyCJRn z)`neesijfJZ2_~0N_7*d#q}+zD@q&oWcORKeMS?h%3Hl}zs$FSP0JLw#o61!E^SA$ zX-s%k3ZQ%Th&IPMnCR&}7*y8N&s)_e3 zixjP>Ykk_^5dJ7u({gvT@&xA2{FZ-A+{%cXqkoq%L#av?qMa2KNKp z#!K0eU<~Ta>SfR<;r3!who1f#HY&?#H+=KihIW0_pqc04W=^Nib6+0;?@{QHCCgH~ z7U=>7Hyp~Moj3n^>u+?_I~5`}{MfYXYdS0K>uW@-PJ!617b-P)sFa2OTj5jZG=ooV zA29fo^_OK8EilNGK^gpIK^D)G33;TYwoYV~T;j%aR-TLMz88E^*nfoD(WSFbLo0_L zBh$4!PC%-t<+N3}@mmk4S=Y7Bp(!^`1pZOGHDb3-EM76ma$5aGR=oVn(U#)trsAaH zo{b1+#(Y+0`Lt;1j=p|YHJb5$jA5_Gc8UUwKZjF{>pXV(o=p}W2A5wUj|zX?WPYrk z;)6l{DLJIa^@SnHE8NQ}mJyUCKLj6)ye}vZ`N1s4?za(gP2}@SiIfh;KBP0|p#UNm zl}kh}W`>*-$)g~v?(;Q^m zleYrYQc}%W+1ihC_s;{CqOY)Tg?HhrnPqbs@a5>Q5lMoQwO0hmpfSg-AwXX3bFBeL zf=-*SDC#`k6S94KJNiymz8YDLkzNVBrYXz2y~V05PVH3V*{9DHP9vdeI&<3F6?P=+~C)d)F% zXJ8P%od4Ll?@rENt;T=IAV=sgBk?y6#YdD+Wh8xnehQ{6xE55R@IK3L-(tQC!lpN zB*J|dcL#kHWFgzILvdj4$#=qy(m#%d8$yifGOR2@7K3NOi}UDpQ|Aqu;R0*-_6S-F2FZIQlbe7 z7Q{59W~*ldQs++O$gEWZew8<$PH|I7xGiV2f%aR|iZiQgISLPu>(fgwC% zl5gyA7k-ZJq{^Dao_x+!IA0^@J78>4{~bV|>kjLYU{K$m*31+l$5{DNPZ_fAM-tJ0 zCt7`PmbS^`eo2KjWxv&TxYTNT3Eyf~s0krpuW1URtsKk1D?|i&hc2=n8M3B4Vl{2J zE(BMV14JJl>jcmDC;eTi1OVS-Nm{HV3g)0?SGxMxf!jGpwKeVQ3c;I)PcTtb8{9>o z)xHn!Qf2a*bR>q#f{pP%T2@A|QHZf-=HlT_E8xZAxDj!;7R4C9@ z)geQV^|5azEBIXYoC$CLeLg7(Ec zhckgU35EH+Cg~;QHp~`f8bT(ESoeB(mp(B$ zr8ogb!AJ};^(C8nuOB-nRaOjEEdkLspg|ztu>N;u*MAh8NjRI`X=XP?u01T|SLOm^ zi7cl78Bvg8))ZD5lHf*ntjha~E7z}FU%X{)=3`2Lex@oPlOQ#8)GR|pv-X^gw^AK! zH_v|dHFYD;ewHo&52Ka;#E9}E`;))Ey}=JnU*ykRA$=I}z2?)WH=rNZmc4JXr_Qd# zEgR&pQ`d7SdgElLZ}sLCmC`>lJ4sqc6s1(@X1NOOelW^M_^nOqNylzHCS6^+QbQe& zvaLewiJ)!%MTDy!+iq^!bMqk-cMio(=@enS2jvx;&}jI3T>UvC?)0_NqjAKS z`#W3c>}HM4W~oH1$&c_+t9rJp>*3#+p7nN_`2L>PV>d?A)t~BC??!xrM#>j^OgFtg zj-aPaTSWiaP-lNxRtKwTi|h!QSWlzX8rv7I~ZiX{ryoLQKya#-556`!Q z&&5~)3*a+ae*(*OWPjhT{#QRavi`#HL+_t1%T0ipPmxCB&xH97_Wc@Ky!$9L4fiNB zADOhSxDai-Nr)%|sIP4rE2sxc{acW$=|bPSMx7}g1?D{Nej!t_@$g15VuOZ?m#AX& zMekUxY=zosfM>MfX)c*dCVL)5u$R_|=yDz9x=z;v@p}Dm+o)e&#W0<7o5VzMKKe&H zj~1+p(+imY{6z*YMzP`?h3h{sJJ}u*Z3ww=NdIjzns%>wfio>=#V0B{37D zkOWR{>oKx4io&h?Xzg@?08_0mdtR_0UENI2UX}H>>lo+J?&Y=9i^HuS=5y>B?o(^9 zpd6HF+OBa_tvS|QyX6>dWc5J0n6(IxL}-w2`Jm_m@t>|%ZsDvH}y<8=_6Sj^=&FkM zG_noIePz0njg?rL9iQaMd8uVgL88Jw%bf#I;O9}}-bs8yY@pc^c#X)_Gq=4WnVO4` z|5@?}(8X}1xzR|o+etjiqn$N3;J)sP2Fs8uBwgl#NEZ}Kzly-e)+k{CmA-k zCRAEv9LMkvy+N+PJdNLL3LsLYs^3p1+;8G>Bd1~2B3#`(lMb%jq-HjH@NCZ-CWS7G zMBWzDgIJ~qT0s(ZP_*QLc})c%9dAI&9>_A#>pXp85I|WZ2DOl@x0D$lBV=`LR;Xw0 z)hKCJ(?JfJgol$W%?VrfFJa@r+2?m3oxc$AEjP~l%uI~?~uq&Yc)<%^@E?MJ0TXZ?gv3_XjeHz#W)q^tLr3YnBYWT08Q zr8tqQ=UdoW@E)7n-+6#8=p~V)*rz-A)H$hMIE?HII4JJmypLn4#vpirLI-%qQknJ5Ls@vpOt$>7oD_ z3h0A{c;4`WPapv3;)+p-gz>>M+Fc&FVipLURcjz~Do}f}nZ+IM>sYw; zs9s3--Sa)%=>TFeqsJ7=H@>ds2_u8ceTwko;rR*TbCW2j_Qlf)hU-~j6< z1+1QFMLFw;L; z@^(Rc@ASUFAIPk4Nm%uwFHrF4eWG338@~%Xzce_giq+2}8@a|tWfGOlN~z+1?x*?U z9M20y_Qc(MnB{-()Ss;7%!gND`NH?YkE}N3+p|E|3>0!#etn|yAspDPz6=bkwL$MR=I>qJ$9oQj zu0JtxWiP%Hd(&^GTb)%a8}V~*EA5M?_xiOF`E@4GM*B-5|EY@im?}cIQAOOjrF%KW z6Qw)T&m`AkmFXjndE>(Mi+MQOUuLfKTTow{U*ERT>g(mMbiOCJxYa zcqfQF|2p%5BA*_<+PBAnj|qIzgpb0@ASn0$CJ&U|Gq$Ry-fl+cbq%{vdtG067l7Tb z&Zqj+uWA=l%I$nnpzVBT(9WeNK4=$kDV2bkz-%CW;8lE;U0YQVP8c(-=b(Jo`$vJY zSr7M0wQ1NK{z2SEVO>xrmQjJ`%3phB9=NogE#Mu>jFnl%0>R`)rHGq<>P&pIBry5& z)%Lle+TwVhJAPfL=67X9@$S9FJf7O;hHHyk)`d!Wwl0)@fHj|y$|}@Rf87qfj)!Ls z9kvaPM^OvhTWhN)JPF)04hHnCnEPy}L7V%eAK9Vj$ojzDpZUqY_2_|T>Oli~PBlJu;t+kr(vJ&@$VW{iBIb5s3SSuSlOhMg>ywGWU)W zCI~!kPgg&H5#P9&Ij^LvZnRU+{h2N=ZhCUXnbx!!AAYr@sPh!^L3?!?r8QmsL7qxW2;XB&E%S)uclMFP z`#=f7J_-J9dNRCv1z151@8e~76=WOg{T^PmgzxEXu|rqxDJcTtvFj=aLKku!V^qUq zTbdpUuSrriY5%BpCy$c$&uectzrU)z)%^apc0Ir8>J5v`DE%J&{1wxq@!@;WH0)5E z+ZHEu_=yr$`WfFdchANjgudTI`v)}Zi-{Wl2IKJj!{Xt4p5$*kacrfpl23|M2L#F4 zEe9%`!MNRWTTwjRh9+`|9(N%ZZ=fdP&mbOTEqnACE4_u**3()`>E4BL_66;U2dJ*U z#s1`h%6L=n^6vrl&@uEw0m~Z5HUA%xzb8EW z%A0sKSAWoSas%1Q$(D(9B5E=24wq5?7NWxd8+&9ZOkg;AWokblmMryt1@{iEuEa)Ges zwMP}9uZxC@)fsOC_x;R;6LN7uAt3_LONDM1yosnU8>EteP&$JCBL5ZA`}Jfy5WPR$ z39aTbL+_VyjeBaUYB3k?m1lYMeye+|@FGX|1(n=?qVMSm1EEH~DN(_^-bh~i-=+6c z*>d^M(0ef(@tM-jafZ*7qvEhBo845(Q+@%9Vl#po*1gcgB0VHrp_gTYYmZ2f|739E zK(Pu0))VkFhDkXF8w_yo!$~#^R?GoI?&5&HItu^97Ma=V{+((^Q+A&8@n4&WUi?ZG zeYoqq&i6S_;_Oi4yC{WA#5s>PQFwX#r$pjyN>@L5CBap{6*9mYwJ#29%-arU{5>bV zksD6le&Hd*zUcuTZ^vb$Mau6e?mQ#l@l>uJOf)?R?3dDhKl#Gd{_oU&X2(15p)ov+ z@HcWM1VD~aMF0uF8ymPc1(r6Nw6m1aP3A{6j8WHivWuvZ`jg4e9$J`Uq|=k4sgs=u}SuK z_jLV6sdg34^xij&#AXdVq%a_}zwCo_3?Mx^3sSZRNK5WKBrF>va8?^|j@Nl5xyZ`R z>FOgs0X+NsE}M+LD1`oJX+>iWtWF1_G1J5A(T6|IH|zW^h5OF*NCR6R?ZO6G#{$!z z^Y`>0dj11uln9{i*;Bkx@}(aS0lhN*m@%ZQulyoMbwOh7SM)JKNZQ0J)L})zJyUl0 za%5y7HgG@Ea@bNwTk(DuM27K3JyBXGvicP`v=FfeB%^O+3Go+CR~I~|dCw-Aqi&*K zD5$8aO*`?2sp%RcG#Lg+a}1(rW8J)obd_mao98t`q8tRoAHAb!j8_dN==~P#`nsA7 zexxiBgS*CclH~kg@%8whNaRedh{)D|Ci9#a=ra@}8SWdO=ou>R!or0dzTN#ekCpd9 z<2jY_@Ew@dRT)psuaj{8*lFItEX_-1QcQpH{rwSjKY+V`q&0lEE6o`@=^-D*;=lFi z#{tqId-4sq0;(P3p&AG~CcbJDC*XlJx)H7&Yj7>>k1v34(2wt4Vdw{)o@x?9C^=F$ z2yZnVCbqk0MN|wZMedI@~##pv%5f_1&>1>E%P5HlPOZg)+6g z&(vb@s8LOYG-5{n9dkG#FjgwE%as5Wzws;r2;PwbcsHb5)=>{x_yl&K%`j6}zmZH` zomL8Yt^dG!)@U;8IRoRjTF)Q6TjV$3<4FHz=z(B*6U#zC^PT3Y!h%OJhZ;-abvLP8HT^g7zQ!-KV_qYgJ)$=dY`niB&yRnpE`AT;k zEBZ0Unzl6BRpEYBKG}LVH#$rq@|>{xNQrg{jVy=UAihQ2H%!lXx3X`a!QMYuwjvD< zyGKgITfHZ|>Q3{#8Lak`K56gX3Yp?fec{#X&3m~)Txhj~?`}R)zMLyWvf9Ojz4(lw z$-sCh(%eJBVDwM3EA%c8Fj@+e5K8&P9v7Ep+|r^)Dn{NuzQ61>CrRGK4Z<-3M#ezP zPJUw}9c4I1N(%UX>I;=6Q0zqW-QV$%$8S7mes+8j&9=yA`v~_@1>>7t5jc-~k)V^g z3(|jh&RQc__zwf3pLt?x&Qu#-eIvCFRZv-`juH5(U0oxUdGypNl_Y+WQ;HNm2CGX$ zCjoF<-M>yo`u?+-73dd`DVXjoL`TRc+Hb;I8I{FBAa%c>ujNW;x@0-U)|EJcJ>qX# zjj+EI%#Q~@Z9=iba`nQ)>48#T%=Z{J_5Gdp9N%g0HE}>^f+x3JSq}T`Y!cfTnfU>0 z&F3!_3Lasp+@O*s^$8BFdf8^+YO+F^d$hysqYUW1@j@?HkN7v`?BD;D_M~kK{&V_8 zf%v}7BfY2#{tVx1cz@tt>xA4AZ}onkm5!6K^w*Xzd_y%v(Szk zORk5%^iuvF>}JL**2N?1=GUa!@eP`Yd(gT#@*^lc>gpdgzSkM+{`ffdZ@X_0cXQ@W zN>{JA2+x~==1B}%Yb~=mYv4UG)n07R&qRg$@usodo-o5-jI8^^9|dJy1=97;tn%Ga z*nu$X1^GREcR5iB;|-du;j;7Ox0LUHpRPXhV&s8s-79K}I%~RbtSsuPa%N4mTPxDD zCib^h7CRrS7#{0fJ`szeJ!_&ftFmc#XUutY%Fd>XE3O@vW>IZ-hczV5zxpxVlkm*5 z^iCL5dNbs5vJ5$vGbS8qEecO48@#i4xa57k{pEZi+Xcp?sV90u=wiSU3kpU;F8<5V zm&eo9XA>FKb1>amWj9ai=U^Aj>Sfwp+P9i50IdV}^20gJx8Qpx!92WrA*R;sG>AgU zK4D(<@Jbx^>nITcg9*V3yk^IvnfFww@&+SCGhz;|LGW_Rr2WGFoSO80l6!@I!%jm{ z(-TW77OpSd)AYo(uQ^+-zNgEa*WEK}4yW6kqm#~VnD7KEWmU-mE0u^?Jui~bqt&^f z5@wnZcWz9rNT)svYHydHawgsyPhkvo$;rLWV{N6q+*y-%lBv@w-PIh9lx>IuSW0a+=RP7I>1b9Y=PerU&@0Cvt7zOo(J6G+53)GNLJidk z1-%edW*R<}Xh-3yNwdV}wPy%XyhiGaB^Y@-PY99icvjrsEkwz3o!Ulax`YQA9yaKn zW6e1kOEa?g(a^jUYV5f&Th4kHw##)waSV1do`<|>?&xn`1Q{>|7TR-@NN*Zs)X>eYkh{ED z9oy*PUJgG8{P6r#s0r~>l!T-Oy^s_7!d@ma4u5t0(U&+3IJBy#7}IlvrYPD{+a8_} zkKu)9Tu?2C4$bYbQgfp81~CkhP1D5%O&bI^jl}G@1wo^;aR8{qk@FYy#}|oGYaZxB z#@=JW-sL`cX@?@h_ovY=@D?yrqeT8&1#^%lW4al7_`HG*oGgAYaSBFuZ-6#Jai*bP zQzfm%L~xxeI#l=eOx>jK+v;{xnHq@EX__#4r#Q4OjTeUa^}Fk@JJqkV(W^72i3wVq z41Uz-sHt(BEWg>GBhP;q>Gr~yq{cAdNw-JVBLK(1M>^|Gen25a%$7hr#$6IuVuCzc zt_d;yN1uaE)U#w>_-=3oEn{|&?}oC=z*`hX5jg5xwRh11MChndHewZ~3#rhMl4jU! zURgLr@drB10-a`oPP0I#Swg3Cal#BoFlDeSuSI3g>IhGm6|?LW?UsGLJa8$MhfN4>bDfrB7*(vBSiR};R0I$^5 z#Vw7?7OC&yRhZg~VymvI$%KG0q)TE55B6-(qPOb;Y()#m%i1?o5?l4KV!>y(hwqL} zg1a{8nnQCZ#n#m+j~MP`B32jIXeS!RO_JIeNop+;)Eey{-JdSc!$le~EB1khq*=|6 zICCbl?O@NGXwS1^>#k-U&s&h3wydISa?-vDC-s}Gh5W-~+LAZ6J#PWU7RADMKQ!!^ z)b~5wX7+4a*4?yd=r#tE=~eh{)@FFMIf@QZ=|sw?ZbJovBRtLT6QW*h)kEP`?=>=2 zJmOd|&8yhA6K7l<;i+lj1Ez@pl#sdV3MLmt@?PtU51I~VhJc}04wHTemv@6Tn4Qli*HZ~l(Sl$fE^^E<)# z@C4L%{nOarTvgORhKnt2GZLzFUp-nfMMM-9Q78w46tU61pPAiQ}G+BO;oT19a*RVWpcTf17 znVbblJWZ4K^$Re2cn9poU!7G`99~uHgRg&fTX6=23kpH_7Mbv#!2Zot4eyE@f$q5O zo03Ic;kYEa*%mOJ*grd2>|9|Dj|Z@l>EZ}=yp~iqM^atf7wcqqo@f-_(7E3U!aJTP z*I|VUM`p()*BQ9qv|#8ES~=Hu77y(=CK|IgYEVb&R=*Hly(jl2#=27Uq^1AV4 zig~_Fk;VLh*-0Ovebn!ty@>pWLto+3(f#8++lG+O=#n@wVetA$!wX)%i7T+f?CCtk za>l!9^0|3pdX*8=|CoUA*q@ywBGeodA_kTOTYkU z@luzbYhO!umqlKkSyS9OFLgnk+eSmqyqeU-_^|Bme!8^#5E2JT4Erp5aYbz1;>bEo zUCvf}@kF*EnUX6a$OIgfA99RnFy@72>}?4&;5c+ZjIzgdg%;wyAui>H<%H!{^ZS<;t) zUyU=>7+>Pi{@A2qcnhVRozQJmmi4n#+m@l4${Y}_zGpy%Eki#V?>)nClRjlF1&uU` z{;&H>a`##J+8LX!zK7=xy}t7>lZF12eb4zuyhT4l`D9+CP4M0>WdL4DuYnQvDDc*c zXTXn@=rZz=7+|`kQ-Pyo07S<@4;O_VL*qdtzpwfc-oxpAdoK$Fy&t*sSc$jWsd7{a zHhn`Xim)~86RU4W+Q!mZYc)(D|>;TZl@zB<=Wq=y#uslVSMhE?7~=JPWAyfq~K8y z3w-hcEd#L^84bjtWig1#VsMn%r5T%zM`eE%75s23{N)kYGu%1IoGrtgJMQtB1)v{| zIhW&;Q^=RUN(V%}&;b(8<#6VK>{_zi^l2IcXdwsv#fCG_rsK1j@Y&GA*>(IubY?gD zoEdQ~6*~Po%|tmQtX+bV&zWa;#MWKE$m7i6zN;{Mw-!sb3%UP{;GJ+TW@*Bqcepov zQf>NSZeKpaTaN~qJ)1iGtUp=p)5E_^yzy&L9uIjJ((%!d_X221VDf~&1@^Cz_hZ}e zR5B{s(D|e@&Hwxzzi)N$PB>)zsrzurZrde)>MyO_d4~L{H>aO&S$kxw9oZNkU>-0D zuJ7f=Xn&bztt%mI$y9Y26}4UYrJHYEQHr}44V*FC`O+6-XUfxiOFaFwQ=**f#a$DP zCCb=UHu}#xpSMdIdc8;MYxpAgHS9 ztAXwDQx$4ulLx>Fb$->u{o~{J$^MPgPl-J3Q`Nu3(GRq%Nbg+lt2EMkvnmY=NCg!4 zv6xSRIj>%=$EfnI_1UG{h~J>Izs=bx*MPC^KR$t{*Fk(451D;d8v8o_{hFuq8t!gMTIW%8#c|FN?q#ZLrzD=U_lWIrctyL0CDQAs zELk_6dt;t6=C(uEYTwL#={a!_557R;ena#fh(s}Cp&mo_8QN>-XPFB##)F~Y1J6)N z&T4I28&KTO>&$Fs>IeFAm!g1ahVzUm)X=*|;b!7Ox&Pi~{(Q?cf)iflxPK>L@Nvip zj)e*bNiD4?m|lxUL`j^`H@@P{?s3`Ktpy0o?qm2rLNzqG1I_hXLi}Rd5)A=>LY%;4 zYy5AGU+=nqcYH9@zgNFp*grXD(!X=xb^n}%{JUGJ+5i=SqF(r-w6u#^GeKCZK$ib! z?SHkhkDO#CQ`VF^%kgql$r$e;#n9SPV{O#T5y)ZM$^7Hj2Xkta2`xQsS>!r$uXmDr z{eUOG&#kjcx4Tl5blNL!iar0)O_1fae-(qQ&vc_1KX>Zz`+`~Ut@`qMNwzQQ17kln zh@T_m73{z6d00zvwCllMTKK;3K6a6@1f@PFOK0fGFKJ10%vviQLY7PST37X+_g#E- z?m?`a872+L_2aC22?x-B98b1)pZgp`wWjRViUwI*L9Esy_t#*U7DQWx)$p+UY`G@; zQxWVn>~^nSgl07K4@N!m>``WG?LkNcA3wf!8afN+x=Sl^uo9Rq#@aP1n80FB1sK{< zd^qhb*m#r`D+_R4x_az%**B}JnQS3>UrWf{$ zlPLti$2TQ$EF!}kL~|~P10qGa-1ndnSjP7pDh;ptnHkUA3S{5ay-O)%3gV5CZf{&a zjGgVmitybt8|mmx&cGUTah)@hvdb<@S;g$7&WS41#Y`lK=0?l@2#))d!s^=Sjv0D0 zy*@|O<}%7DO{i#;Xl5P-C$lWVs}?gvx_VKFt`1$Pn`|Y0E|m<;KRLO z1)OUswOv*)5hr>LXzfrevv|1N>RwP}lv2Zw3N=kZSL;!eZGmgoeo?q)5=Lt&Ib&fR zXR$NTIkx8zr|A7V&@<4Ct+^o@Ta^y4{=ElJ#(EwGo}q^_1sZyRRa1ENugrV69h(!2 zG-r#o?GbBn7Drm9D1(7j>BSYt$1y)VqlfKaWCIL5xz zXc!8>-$50Y)H7T$rFfu9re=Id3}5H4AX^kUvzD49O-TIhKxn&pF$^Hx3;&JP$UX&! zzUn|}XjvK!Tt|o3>0s%WbyqeDq5GaLcMrb#CK`EjUEFzo+Giu-HIkWIoxd6XKFhhG zE>U{8#Xi-zui4kb@vnxlNj`3=T?AuGj$~|CH!Ao*jO~GSg^Xh^0T4jvjn7cTFt()q z>3aUaA#J40W~JOvZ>1kfKz2odo80it*BkUU{La9AshD2Y-kILDe&|S3Vtabm&(%*vd?8*%6#k0q_l0beY&O5#>Vw8AxX5kQB)hrlETJoPpFjD^0(-HDti5}+3 zf8-M{sBhY+^-UhFzMDs@FRy>2(7%|`evDccy3uQ!_&XV?rgGwS-?Qbx{XZ*pQ#V;T z>OkspR_bl|FtH0UT-^2Vcm>tDXS@N;0S2p0Jz$;LkI@E;FNS3KwMV-@#`k#u&|~p` zo|_3W<#UT5={$kABN=n!%F}UoOw{!p!xG|a%+0BZl+{RP4UyU7+1cI~58u@u?b=Vq zk9zDajcxXV`d?}nc#lGs$F{7Q;C}PZ^f`RDW+7ws${~o+Z*#ea7i&2ZgMApXUTz9zZ?)h z?Hi)fJ+*^?hq$wC4TV(eKR;upNX8%zu30mVcD;1Ct*LpXt?Wx^x&!_W|G{_ffh06K z>);W8+Ob!mANAia6{CjwBnuN1er~mYoCj{vIzO+~{CX?pqCl+7LjKNH?+JFTd>rJI~6cUz8D6yoWqtpN`1Vlfl}{q z_j4s#xH@t!<)9LDP92!ylj%o>4|6UCMd#KVf2uek0W5K276TG%Cm0&xTf!!CSfMOo zy(-ePX|bUuzV9UDscWGj-k;}y^&GC~YW1#VxJfbZ8^d4eg$LzJmjQXz9FgnZb`a7k z-!@5e(aVP3#N98iLfOi8Qz-Kg`x$g|%g`tNXS%*O@ZMBAG%NcA(Iu}7c*Er17x>G( zlfHYeq_}{`8F+r=iYg8+Ay)0RP**9tuKO*LA#H`0JrxKVSm!LasD$ zPYDRU(c{A^SBL87`)7E=_wzKIsPCK9t=ZD?MW3arw>!S**0ZrP{=vP*&;!072)$ol z@SmBOTw`1@D~E0}a|u;6q9H-v+1Z93Z0>B7(0jW6nWTS?K>Y1`Ki}PVJF#>pbJ4z% zi(|R?-VeBlaPd36l6>@a0T+4)C*5Ky=Gkey)U#4O@cAjn-jWXhpjoL&ec&8E))-XX z|A12=!W-J+b;{tMJrOytrM3c(omw$8gU2t0BgxuGctY_@;R*5w@|=(;4+){Q_o{#X zNRJ~KEX9d2kSC|InMC=BT?w!X`8F_o51%BpA{XKWVmyzDOIi7RufTaFu^m{0tA5OM zC6w%v&ZnmLDPJmaB@^5;P5)2iCcNrzd=QR=oM}+rv{Op^RbS!SG>qGd_K9dFX8fVF zA&Ugic^;j9ZiNS~DdEV6DMODtP-XbMrI!3F+);y)=d+t@#sL)C;I4!%;p z78|f00<80H7FhQsor!=|c<~(l>Z)@8NOzNDgr$?$z%lhXqO~9hGQ?=B$UO&7>m&JJjB^nf(rzZI zBfT@@mk=1tpL0awZO3!d%F)MDo*U2S-(ozUd`+hSjAJ<_lgTdyKEIw%nW8v;IH{_icWX!orP7%hs$}JE z2J;1fwure2ujG*_-7m_X0(Cm|5;T&QMvBPlc9! z!J9D{Dk7nBhUnKPsw3d*C&CgXMf0foC~dIhS?VIBFlis3{=#6D|4sO08xV)$$mv2S zh$aB6n#rwzYp@3jxW@ygsE|(q0g6E)gI)$r>P@Go)9I<-_Bvf}xPgJ4%9{1xL0qB1 zEsty(av-WpGA$3Wx@qGr5X&F0zaQ$M(m}@Ad7~tvyhT&xrj65x7ER|&Z&5JuN{<3r ztEY`3)&#Cbp;LVbpWqEgv+E5F`OE@+-i-42n6SFJI57W#jS) zJTl+&SLBi@pl{wm%=zQLLZnSK--@dM{s7H)vJJMKM-+aM!f~^O!t;9*sS7C{x?jcZ z=3WN*Ebn=8CqF!aZ-oBpUGC=cTVA=n?xEhpX|_!##=Wz^84b3Qt!YRAvcy2>a?0sH z?=8sR&5fMI&VGsCm76q_40d_)8Aslpp$-DZiS*)FuG7^=>4{S{Q9np9*G1g;Z&6PN zLWh4Dl#M7q!~<0Lm-z9|1MND(6WEFo$+VFc*hVup-~lmFPmm55KNb`pw)G#(SD}#Y zKE5oFzmaTfIy&(+ij`N*Qu|U{-}V47Cj7vQmC30_m^yP@wiiYqJ|aBK^m%AW+={oH#1By;K1!gCIMHMXX#P=lM32K|szZEA*_(3m6 zPSBsXJ@1kC#KWFoh>ZE{j$*R}0}*V+f#vA!k3?0gbI9B8LZpA@4Xfw1vd%Js@SBaP zVsj~V-4*V7TI9Jts9#x5IlD%bKUn6)O`T~_J$Pd7p8{>8Lrl%WxGDr2EG$$6c!N0@X-?it_d4APs&I~yC) zzJ86i2A8T55jkI5F5}0HAT~471BTe7$1UHfa>r0EVZX<{{dagjHphxZD5(A|2!-@P zTF8(YQ4~HN-sK4`1@eWFpu9W+-7}tD;&mE5O^G!-&u!SV9?CE)2KPB4w(5nhrbOz3 zn0p`6cwibt9%tmp<8~E15*N>;{eE`^J*KmoLm$&k%!t`Ek*_R*4?0e!4RzTkjC?%& zuzH+>uUT(cvDw5!%?_{HL8U#1O2VtBKngAAgJucXvzYGkK>itn&QF7{hepM3AGLGw zFG2iHY8jvZ&iGyNfMl;D2t0~&h+)BsPFJ6H5(xj3e1t#Z&f;|hWE7Oe5bMoec-4&t zWZT{8*ciKKv|Y>0be`FO?cJMQM!D7930w;%;+!1nDAt^#x!lkEmgy3@2z7vu+{p8Z zP5@*0z{n9CcdbI^nr~Q~qk4xao=@vbF{CGpB$PsH*s8*frKku#(Br8eEhfblZ<>~tPmhXk(&@UWn>i12LaDt&sdy2dv$I-@&4V&FRD)QT3%*5tEr^3 zPDF3i{0_uJUp`4YG-Ir$as@-@QxOjdbM!jE4@c(DJ5HI@Ca{__`SU_ZQp4r%BMF`n zLp?ca?!xuz%`t`qSDF3jKt2!pv;BAz{e1L_NH&|^7*_b0VW`|NN&j<_);)pl2Bm8D z4T&S$%*iPU$7;trn7lYcmjH2pO_}ldtmhEXRZi7$KaslhD`JR5s3y2yd;}NxsxtG` zi*HgrP(fd~gU4Ml6w1^XtPi+9o>+OX2y$`#Ro+7DA zMbYTje%0J5CD}O39&ahXOt$QOs?I&-$v2ho+{S@P4i&)y30%go9#8n{7?gwWE`f)$6bYfbf-Dq?c$5pcF z!a#AG@CS;&Q0K9p|A8&;ahDl!=le`-90uzx?vMWOn+T8To^`atE(@$q6%3_#EK|C< z>DG;@*PqPH_{`86DEPQ^VBD>VOFZ74ui`p*XHOTWI+)RobO^|f2RYIGg?125YpyWL zn6=i?w7B2$A9`Lkeg5h8rlVFins$ciebEcf9qm0Ox zoZE^sgCs#mN#ed=iUaW%zI)CX>)ul)pSzreR{ws=a>`DLGJ2mR=GL1pe6WTy5`%LnUHzw{p&&Si;!2<`=!dw? zR$+WIaU0F7EsP$w@jllWSGoF4=p{9NF~7zo?@;61{2EmsyC~Q|J)&dJse4e9X)lW+ zl@H8}dKLoQU{Chi8Rz^_9k0x9M){{Ms7Y79P>l`nRlJ(;GpG~&n6P$fy`1R?>6lyR z{wuZN^HirX#w6kykvwOu_r~YN2buL^FQXI~FnfU4P1fpzU6c9J)SzQ_SErY;_I(F9e5(+t+a71FWha~lECOD9 zR5J6PUH=|F*3cRE`6u5RA2$@>bBiZHH1xJK%_LKn#c4Ta9c7S8<6$d}^JQ{Kpo~e+ zp|7VdFUF5$b_98}(lfZ}`Yq!r$iMKf6Hz_3IrEUw$y-rwr6vxtJ_arObl~ZbC@9>z zyXlfjl3s>e_u1E1^pk-|mSF>nm9GmHTit29J(ys1)c#vOmJGx@MCzHfMf@lO8E&Ue zx5%;qTW)?BfZ?A{d{6wdQ;|r9f6mz={<+CKz1+knS)BY$MM3`gYi8YmNc1u3aftwG zbzQ4=>hCRzXeN%uL8E6CJ&1QNeIEu^yysH!7>63LZ4$Lo&KP{iBDT5)9WcpJ_K2Fp7+b3kkblql+X&+lN&;vci zcM(ohH}t@?0(=O<2dyR%B~cUG_vF7kB&f?7rt2J!r_cX`o?IA3rFAcoSFyQ1XgOc3 z(FZsG!rRHi;IfuhSl>}k_X+o{6fnF_XX4<4_?C$;fdtcqc2@ZP9*B0hRn&wVmQW5t zINog&bJKPfle5*mf@-^Os4r65v#sugP!2L#Lyj4rP}w6QIeFrjjb;+~Fil+}#EDPR zkou_aYy5ckq*H7%7fEb#Vm1De%t@Qmq2>JenHXuZVwQbloqc_Dpw&>yboJ{IL*U0b ze-N4cD|Z3Bi+v2F<^~14hX4&PNt?!J^7GO8(1{RG z-=&sGINQ0P!&o|SRVtEa!EEgE;nlC3=4LIDd>~Rf5Hl2Y9eKr+EKK`i?wCoOce8nI z)vMaIVZ#85Jm^lC$gztLn6EF0dSnzjv%qpbzeuurhudDE30&d%Q&=kpgop2r-h2=^ z8hS$}utIXvqQr>Mn}%I2O|v0=F=)XHF-#q$edKh8& zf<=&SytD^q`2>eRXfqb}2esSL{J4kWivVsclWkD++ngVpNjGHfGqQ&8%h>$_1YR47 zY|LpS#G=B9oGq-4R8l9Lv;A=)|bqy zhCwc0&Rza3clk>0^3~ks0sqqcOuk?#q8;gpO{$)<4eSnYNv0-k@$8{(90yFW{opWp z6y^3l78$wo*yJ=TqEtUc!%roe%;_&nYlgl7I$A4NEO4=tT$e~VH3rVL` zruYGKak5-e!I8l3*(Ye|JFWZJ!O?%%-DfSZrVMJkZ;A3QuOobm{Hpks6Q--n&5y?soI^19~d*?@1YXt1G+SbM_G#IKYHH7Md2J zBu7YdZas?r8n5Hc%xJ{Qs!lnziLf}UVJB}f?uLKyh<{xYxGoJ`mj$jvf$K4W>+&{9 zfnHq+MSAFRGNy<5jLM|6VzoGxTP$Z1>AfpAfaqEzy;>x)x8nal*m+z+q%CgN$=G2| z$rzuI)WmQ$w%KIY^shza0{<1tv)orm1J9l3pa%GV111QdWcIffD93r5GjXsj zHEA%}w5tnpu_cd+9F>5YWC_Wih<)nnw4CGQ5M0mtNXoOsED)X3O$I49a@>doG8+{l zw?IBkwMqPxA5N!<$b|sp3m4sLOUyZO^6}AA?8ow|Btz5%Iumq37WTwx=9AnDr7s z2TvboJ%yHB?Dw#&_J3pf)cDIsKM5yMZ%y99@&RuH*pYB1fgP2DR?`z%>`(-l;W1#x zxBzw}>{3$F+B{|5aJo8>-oMj8I+Amq08S(62U8Q!;LtSk;F)ia$*maA0Q0VGG2Ulm zau%9$|B`iFZ;UmQGnQc=pIG0lN(<6LKDRm3$~JVBC+uRKaW%(j*`T$~b;2H-#izI_ z=u|?1d>Rlyk8McXSz4jz=4j`Vx3fnFfJ6Dh?~T6L(_3NkMi6H0{fZ8bZa>P0=YPL_ z%O77rek3Pi+0N{e$(ww}S9t$F-QU9Y-|3$I73b`QIs9Z(&)`}-hobSExiFqH7shku z!Z=3!gu%_XHW?P$Lq^Rc@3Y2sc_HAaco+XdF*TWZ_1)V<*>$k1Xn7#nHuK-myEfEKwDiyq z+1K~)7wf*E!rYG%pViI|3|kNg7>KM{V=ZDvfZamnB!EoNQ~&zN!1Ymq>xqHu>cI8U zf$L-Z>o#Y8eRA@y1W8*UqBR6&B&U%J@Ra1V;;y5OtRYenheRZ5=GZM0DwL=u2D>8( z{dg4`W3n1j2DfD;4kL9r)1j+56^V3$Yx2|d3wYXSKsr>)MQOj81w=jqSe_;0DLUv< zBZJ{Go)t7q2xtx}ZHPzz#j+T}e2|%-X^NR3=0h6;%;arFjY%U!4RYq#rTxtnMZ^Dx z^!C2Y@yDFZK}Itplz(S}aOq+=>Y#ORRxqEkK|38rD1VutsC0E}v5`3-$SV3MiARvh zkHBt%NS>8A){mD3$T(NP#DFG{wl2?OUnA(#JJE;2_y5!MLhD}n>p_y3*%B3uv}Tt= zBVKIF(g=dVT(+q&Qp+K<)#*+2?Pc8~%%<82VBLuT%ep_?_(M{6{NLlp5wh%!!J89er-=sCxwC)fM-*~rj1T>T9z_tu^C`mU>pgUC zuIF|CKVSYIArdUR`~^xxt8b7FW+W|uT#gpM4}A=UlG&LSyr4I2P?>~%Y@6g4a{ntt zI-THp&>iX<+2AQ})nL-Ol&W^3T(y<%!bqW*HK%gp+-V;*r9+cG*Tl zBtaNx;Ug8Sv6&Tf?Us#PQ3z}+q0FNh%BInbChs7hv-gluNF`57q+$nz7sx;UeJ0^d zQrTl2ES$h=CXyUi&|f4o2CJ3|<*5<&Ea|_2_6q$={CsEaBmg(j)WuDi0!&}|>k2uaXo9Lw?U0w4UN4q^`Jb|$2Hi4?Z&^*0k0l!og$=B6Eg0e+4&JG-QDl^C3`#X~ouE%c!uB%Dy2>scsrA{QEo# zt(hl8QnZq-Tc-MO*iB07$Q8Ut3PtcWRVahFq$Q*dF{fi ze8O%7>3t6}t=KO4+%L0R@zg}?YBAWGGU5c5(8bagMMl~^7Rme=_q>7$TEYY={x|fz zm75^r+nk%Argxa2ZrM)@=I6G5dF%Q4{tpV~XDM#oZ$Cea%={$CpzmmkivPQ2=$m;n zbnyH6Gj#8P5i@jh20#Cf`58iH^~61%M_`-t`%RH>*m>%prF*bij$}zu3ye%{?=GjiBoFzHtMo`8W7ac6lvsh)9 z7~3M&n1CEJ$hL+DW@n=(&*bEo|7m@yP+uvOhep`%gT?^pvqTk0PhLC%Mk4%T`3Ojo24hOj3ib3Ef@A|o?LiVRPYT!z_m{`T>~E+uikZ7VE2BC%7EQr<{54Nb zoojk^+pB)BY-KH@;Up&Xq_RePz6CkKU(AvJJQB^qDOx9(3rDJaYkvO3cFC`GMt*fd z=QL;Owv})Wq1PDdy(an=S>QSpxE>R@F88m|g3tsvSY%y9 z7Se8wb8!upunktzjmqF^g-<7qYozK?=oIqJva*-M3;M#B?9z)}M8%s2kGvedd_(xGG1q;> zTKQTLWNOJcd@9)DfDs;<^)v}{sLYa5j!Zy5bB{NtG~t#$t*8@A=~5ciK?%nvIX#{9 z7shO?a%^*^6HQNb?Ik8i&_bu~mxy3VL|FJ+poUD~<%bMvbR+P)ARz*ax(?%C#pufA zH<*rewM(EF2g%Pw5@ikc1a>QB-;06X-~?~*CX@_nGm%=W3L~1yn;3)#9m=*OkjA9W zLq_5efJ(#Ako|zQdij~fMFz$h*lq}4vRMzdWMI1m*!G3bD!%TzzYDhRh6e3cdw^}$ zvbKa*?G_-;^Zy6hSazbu$KZIAaHKinehRN?OVAdd;Tu1Zo)>S;yv)a*AxWyjD&5mo zihJneiD_j|bR9L$Gg)6`Dbk-=-~{1VfPP z*J*izr$XbTE>zOcKl%G=E8nLJlg`i5g}R*VkK8ZN8`&SLiX0$Ie=RLX>ZlN;^?rjO zjhtM?0u#~qDQOld%RRj$J3sVczi<-iHwZd|og?9OuELc^gBNpK?A5H%h@P97*Oc4P z$^0qjC2P%0`(OqS%(UlfP?6jn^%?PByOrM4?7+O{-r1hyKchHPe(J;6^~J%J(#F=UWtoHZ~L5i9H0 z=nsFC@f)oM1OQtq5`F^VTn{rUQKkvm?=t(NIB|_=LgnB7eWoU_M+gwP#&}eaj%LKY zRN1u*F#`e}Z<`sLehc4c{O@oh*oA8?!byrxb0Y`e2Y)ME7{DuNKzeMta9#6wix5~ z6TP>3p6+Iaac%>rne^PM@f`eDd_U}-+s)ftD88xC$cvsUu8vkYJXK!l&W1#G`Y6$0I~##HR2ie+gf*XW{ynspQhTQb$b z{ok%*2M9|auEx#%Aj(5;fhaqEo<)>~7XZ~)v`E}!Txaf0KGK{gX$ISOYa|CdLQOt&A|IEVygkBu1@gsC@=bdFs2-d7R z56M|h8z^Q{g;JLNReh7ir;Piw{~UHIq*Priy}I(Zr+t2-d{!OIgc{|&_JXp@d1q~dmH&n1E5+(vir5qj1fgEkB_Tc$)IIo#I0>~y9h z^w5bQimj6cdit!Pdw_vIg8ukvSz2bsXGF+E{m{#g8HZ4RJ*O_U`s37uJ?;y9oOfh^ zztX&)jG-}ZLPy}&6+EVsV>o=x$3X0&s!du|Oam?MO$?r+zyY4p&Y`rRlw zE`NO&(jWM_bMu#^^bEu+<++wBYh~3I0aZm2lMh76#xmX<;&*YppOG=K^0LN@F*&ci zgYt4~vtmMvN=UxkzRPfIjwS3!zD1Y)20vp6LggZFv*bG_9|{xdt{|(rPOG^^!$9s^ zj24GzN(fPlhz7awSWY4LWCS;zXh8}Vo}1HeB%^v$nwqiu1Wn%Z+Nk^8{USJn#*8JCXq6 zV4V3-1ZSJAXjfHF`ArvDsYTz<1$1MD&n)@ z39SU1ww_2Gth*T}D1gtt@qqi67yX%QJ%DH5dG05@dAspI_l?n_&Xc=uJgKPj$lRp0 zotmz;pHV+_wN_WzDQlZ24P@iLK*dtoBtqyN6v{u@^D4-Qy&0h2u5k~v4rErsUfnyPU!R=;`l%*vXTRlqA&N(LcolBM zEVh=^%yui)_DA=l+BmXryoWQrlJ<=cqHxCU^T=Y`Qvm!~V0;fSevy~yWa``(j^ki1 zy9=|~T;{i}oOuengJV@k>oG0%jVH=%f@N)Bc8Bydrjmo9XD`CW5q0PPhq%kf?n&xl z>Y{sCSZ-UDPQUA{R-G-4Hx_p;q*o`pYuM=>zKlxBsDzV@sDRYi9cqVHLt@zQ>EXj-K*}d z+;lrXo)yBIS#LE){neN~*43Ll;eUZXMUiS3qIIt^Oe#|?ziVx#vggEg$mjGTWFFs%5rsJ_wk!i2kf~VKndX55cN{k zoEe2ez|jM*FpS}oi38wQY%wc~4_9S{@IKyY2TAdh7uMMq*7IXYo^EdE4>H-@!QUeO z7W20hZgz=+LB;s&-*gx8anD=}{)t|(4vxv2xqNtGhcS=MZFe`%G20P_d06oBJe#QD zN4a$nd+y<$guiX6Srzwpb1g|as${-V_m>+5iNOPp;su*Z zSeC8xHsktb^5bAl9E^#BG4Z8I`{Nb-M)+f>aT<=-@mJ5Ew6oKW5jGW5s>5i131^my z(AO3kv*R5L?Q0fTSK=zSw>(igz!N0GhfA%sSH+;E?QQl+$+Z?+00VgSVZnEQie+&a zlj;ulD=({$27xSl>N_BiWYz8CH2uDf>FJ;7CYuut#5GBCb>yH~%I2Tgh33E@)WAG* zck9&b?8Vv~vNgL(?&TbEs@ywuhz`q$6L>Vcbzv#{-}=|MafyclQ+)qR`G9oAU!zwplz=4@iLBK zw+-sUo{c%*u(9#1a~SS6I$WSMWeOqR)VqIrzx=}z_=hFO{<@cM%_$*BgbgFe)9y4K|05V?}>SD%wjTSs88QJRj+

piIHFUHO=T=^YjaPiU8NWl)$la-y&f6hnf+@Ss32>mo2#+Ze4zQ7JDp^28!rZ z{R5TZk&v!F-PGqV{Wrg=hORdaP4Hf5y(?A=gqg>P9(5C~>cCvY?0A&zsVD@^fSxso z$g9YuKSHUTZcl1&sXa$xJKdUYQt4&ncO1LjvzP@=DMS)%lw|J9FrDGRv(j~!(kL!v zoqop`71l`+n!cHvJbOEA&x(qz*E=f(!W_Vip?mr#=^~A^2-|69t9xvP;fLc1 zC6#7m*_R@VAnx+&@BpS2VSzkiobDlh}rZUav^sNB3diUis z_9P|mA$x$)M#xI6{J$*WQM<9_Bt!=>EA79pmrvcRza!pz zPNo%B$vrRd`&M$#X5Jx;8oB3^I{T7(sH>5CX4Bc(aIo1OMpm_FFXmTr z&lQCGaYDJJD=1K0sw&BY?^Z4+R4$C(vZvz~pkUehq?`vU3c_RYfYp-}K__LsDW z1GX2r|18psg~scEnWLq+@XhkwX5Co)0~m?fJj!a@3yFUv0=|jU=%@G5Br7o+BB0}?T@!+oPxf@iOu z!1&<-?woVQMA#T)W67CA7uc*_y^AAM*_v5b!S@S_u%Rq23a=gyej%?qpP$I&%<2q5s#~ zxAFg4Ug-a|3!+l!pJ&gSO5`9zARe>@d0!5By(mYp-)2d%W=Y)U&{vyo1HjlJx&Zh|h34NzK&!&8=)YlC^u?CKJAR2?fG#20H~oan*ZoYyh7^6IY20?4W(G084RI7( zf)SIWcH%EJeesJWdG{cPXW1y6r;Cjnnvx;ScrY60^PD4r5Zih^9`I^KD8xYM5$9jO zLVlG839Fog5x}GbQ2r8{77qtwySKxo9a1ZM@#!=obEio;cwb#v7|*+6Cs8xGD4usY zerZE;35bw9yZRo6Vhbpu+TfI}#EPYFIM?59(|M3QMceQ0KZl^eZ_TsMYrJuM6?LV& z=qVlo#<$01WF!R$1+U;HvL}SwDw8g>yc)$o z78GrsFdWBl3BDtJ)O~v|jrl#n!gQGI#q7YOWCi5$=MYZ@AdSA|Q)bo8#;nYG} zRv!lT)%o*O`S5M5l-TP~Vvn;9EDciR(Gkleu(}_fRtjC4 zC!K}qW0L3_r|Pgp-)B2G^ntH@XeSLt3SXt@Re0__+jHl}&>-0yf;&+wbJZnBz^eIg z$VfZJezpFNjGv}k)QABMZX3MgV8pmZjTm@=i^=l=amK8+6nxEtQ;Z|+=l1 z)HkqC!PYnKImhMV`E!g1qH?U@c!;T9Xt2D2Ns7t{rhZwYws!V-97}u|$!Z9g^r&jc zV(hJ=8nPGV9*-K5LQ+F-KQc+35=_WbaWO8W4tq%@z_|^B1 zOb(C_mS&=)6!gbhuC*4SNX+5f{Z6#Wp%QZe11bledg1}y%rWt>XBj2tkMcQCrkPV< zEwvS9Di*;x9yPZ<01typtuk1Yo+ ze@)5Xd7pw<@|91Ur60g#;KE^}yL9``K0&vyrGUPH#azj?Xt9bI;qNGu7z`q{<>81y zoJ1GnUZY+fT%g01Vo&bnxYGzXP4Fs{Rd|7m9wvEl#Y0QMT7G`gUo;E1H?-q7jtd=zkyg*^tl1&r_p z;-4aJMAUo6T>ze|<#lQ=O1WnV=JhiRs!ChM)`JNgJsV**HX*kYx04&Z9kZwBNh5a_ zj&h0xbyj#1ym`{LE$zn&H*Ax$woHUcq6|*JaYTxSM)XSfF4P66>B273bF*(y1}pJa zL}1L_!o#63LBA`jIJ*g%67TK7GARIdLQHywD4xc#{Q&&6bfGO5WdwQcms^A}R2b~! zMs+H6-_0;G>H7i#kqcp-RyzpL{49XtA3ZTEl#2*);2+&n?hmsI8bM>IM$l8IW@cwb zI^ub+gMW1H=6&txUBmO()B>Yo?8m{2D`2bbBOW!)pTU=AApC>?h*c1u7-(g;EFi`p z_B@o-;uoWrW|%d1L*F^I6nNoPQcx3tAYd3%D3DMSqJhX7cDNC~PlAZM?}Z>dprQva z0&%-_%nhxAFHVGK0Z-GRcqj{oJ#@)7v-n(=iG|(mE$vv_3ucRhq1~N5r5k|hdljZ5 z?q6M?N`|#m)g9lJ=-@=B2f_kT_r4pBJ@Z@((R~*iF7(gf&KcTAo0o ziudYGB51IWBe!FV$hAP4h75j9Yh~v)T761IWRuvrx0wkQ>e(gWO=-d)}OdxtXJkP7mo^XncYfWDNWJ-e8VJYB97g-t-6I zNij)0Z~kQs=8{-sCRYJzWVX@w3UP&UMt?^LE19ZJ6jF-k{qR|AOnbW>!=j=fqgj!& zE7(_&&<%IdSRGZDaG5s}Tj;*;h_Ipb>lX#&OE*y}$!eBd0eL-+9jS7-Z>tL8flQ1o zT6J_~qzK6BZb&AQ<+Rva7@zfgt;n&X8z#IAPp&^3);h?o@_f$#BOfx#UkDlHD})B7 zMAY~q>|_|#I)0Q`_IY2l2&kr8eXh{ypf~8ZM#S%hY1QWb+&1-ovwsi;xVqbv_6G_s z^H*3DqP~l``8QCI8eX9Px?_mhD)6!7^LKJV%*Ap8PI~#=1e6j1{}uUM4f*`QG37I4 zv=+`k+V};9^A>J@gm9ioL;TT&v;P?`7SAdTFZkl(c_zg(e63mHdEygEEKWIuH9=e8 z4z|TJ#4?u1aMZN3m+Vi7Y=CT5LxGab9?Is<-ttlT&*vRb!GM;oqy1SSj~2~1tV0ps zL&_Kdvg{_7C&BuXMl6d=<@#W%5bZ7{b1>cLE+uzqDY?rC-zid3VKa>^bRg<^)BC$5 zEX9IH6NFe7`W%ArdWRrHr(nTvtNFtCZi4VUQb>v*gta$G_FXrKO}6D{I@zasXBpgx zrpUe5AOC`K@2Bl4a?g@*HxLa;67DR>t27c$ku~?7QofN7Ecw=iBDmwnl+c!McSCCb z50YCEX(O{{NBkx#o{wP`)LU&uJ<04g0mRWdyoW zdhHXN$-c1CS%<^@4o;v{^Esp%%u}35Lg~CH16MbPY!$JaJaeokA4%V|gc|fslohng zN~>-zfY;M>at-gO`v%5#;U-Z)tJTiJwlr!d2Mj7bw#vC5m)qhzB=yZH&>lQ%Cg30j z?)JD9rBaU>t+Zp&j76bQ3UaWgzKy#j9vr3=Vx{oJ^E&<@U^25)EAFP#wEi!rPJ_Gd zz^)r(ITE5{kD~ORz->5as8V`!b?nGW@5N`b?Bh{-pBSM^{-R3nd2l@7pqev zD!t{AH;z}o{f0xVTl(!M7j+Zs6OKc_4RC&(&~OKfpPr*idC#)TyD7Gx(NjzcK@39N zK@dsgn?ez5o;p5?SQvFuL@fx2T<%bFhkZNkAi!O%wuXyIw`jPQlLWP11M!gTH8s7>!Y8 zb1+jCkyhWm2D=84m`02wflHZw2vFEE{iYNarnSSt!oke4$VV{Jqzd$p*Q94(OIh=1 z(ftbm5QzLyCYF|gqPuSWxHp|J?xtI(w>3!NQ1n1>Bmj(Ls*j#GpZaA|2gMFMCGYFU zD|x@}ki5@og}Uj&Zj$!`R-U3z&n>`;d6KYQgk@m|&7@PQJxMWq^iCLG>Pt~Q}G zhg>ZZ%~}c6|1t7(&X0~uo*v779dK!Ric#eF?bi!2+6L%YApIH@-Gg50T%$^3)-Jvk zIBvGg+V-HO?A2J(1cBLSieT)-o5YzSC{es<7OV0Il8&S4{acCia<={Z&Gh!~W8vrQ zIF`I=U-?}4S(AGN{9x;;Sp8wX%x&2k#PVGl)a2iiKuubFz@|5zs{mLo=b4?(gjd~VVVV}Zz!mI)?GlmJr+5?Q++y%7(ebAbVj?|~P)c*3Rerq{FCaGIm z@A?HTA0+IIvyl&*qf=Y;1?uCJJ^Ju{|M8KC17Vu^R9O3eFXIBL*l;HP@NExi0 zr3FYMZwk{C996M^t#E_na(FiM~lS-JXu`{bp4u-+tsn$?2 zj6lT0l|*@@X%(41oq;woX>cYs1vHOw1&45B(cq{NHy>;!KDZJ9Q#o^DOj<@&Uup(h zz{qnmtrOlrj?2}xTBDht)^(B zQ?Q@)BC{1|)L3n%3{$`nc0oxiP()Vo-p_dG^$i$;xrxdZoQi#`^bfYQO`5B!xJ0S|Qtc-8O{t>Js)E_fXXMTrAOKYaSVfdu zX~&{(7M@ogE%ULsBJRH}WQTc6)wqN82b^Li0?+xGg+R};e#q}=Ay!tHBC;}(;(bDj zcVC5Vq@|sJOk=C7JT3o*NasBYyCena0S<47>=DF(ZxVT;L9s%%2R8k3x^kM}Ee}u3FT;%p13_M(3X;as z^<5}USa@`gpxYc2ATug1!HOei{=ppU1=$8^H^MGuB3Hulo_c%%`(1dRB*E{ySqb6tr;)|U>)gwKCD?mvfzM)Z+}ohoJK$jNY53j5f70S z*&A&V_O7LU6}+BcZaOOC;>(eFTL^JMGTYW^qIzCvnx{D~yw9R3vMt{W+qx`bJn)6s z_8{pWH0jf4R>}EJwoTOk4}$nN73poP+u+cvhBCYe^wRA zY|AE2O{t>Fs)E_fo{lQ~tO9InkZo&|*f!{3+WI$By0yY?8oijGa;)!A5Bw^;LNyh`Rv;Z@&N&gFd^ zx;3%2I-gy*rEH%X)ZNAbt>cv8RN6l)czqO3R!xOP4Eh8d8NLz1s~0*us0V1sp>SD~=TzWH z4Pog9Vt-`fV1MSs)S-CDZE1$$Y2ayDVwRD3c7mn1a3p@toOzE$ks-gZG|Vx7HAh-> zdLTH0ymDru2pB;H;6YkeVpF%o?NRtWB2`3f`cW9fuIn~29BC~;*x4Q;NvY;NA0^oN z`&OAv&yWuj+o}urOT?!Rgpelopkp=E|-xLlnYs2~OT{z#p5(l$aV>|Y1ucA5O zidG)8{`qQx2(~V3#c}N}^R!nKN3WAKWvwM&e>`|v$=83IbdG*)JkO7%Dx9OouS@Vt z(W)q@062Do&AiaMInZ*r0nWe_zJc&%eT32W7o}{b5RApZ{dC+OYa^7>^7Ym^{cikx5d6%QU-0uT&CiYdEq*>&5qT5CujXecKhU{DjezynA`Gp;*J9vG zFt85{Tv`xFVc;qU1J{GCIg3(C^IX|*p+;Gy zG_U)SMQWN=F=qVLY_(e_&PN zS=I&692nM0^VK#Go@Lye!?W1?ql#B4&50{d3-`P^dlPq^;cPb8uB;1uxfZ;-n!LIX zH~(M{Rz*7*n9N%8x?l=NkP-g&8q%lZEi;`}E?(bHlX zY#|;0_wpWSw`w(^D8f{A0H9bs0*9q%d}rxhgyN8U(Rnb=0g&*$79cE) z*clj7mm6cUfQ#hpx54-Qt=L?l-a?n0hkVZeX2kigC|x~q&plx~D6kzQYy&p7I$RvW zb=aUK{hL4OvBcf{i93{q>v%T8M-c-l;LGM0jNV~c+ji_ojeFL($KV*aO(Nax&+)vw z?v+C*JP@lknK>FL=Sh5t{cez_RXk3CVa8qgzHQqql06lNa7-27u#xZz{0DHd?BwZ( zA(P^?HB5$JflEF}67W0xDTl{r`$`t&<1o$W+p|Ljoalj|mLnxIZkqI29CsR{xn zJl2o{KWK!6`9^p;Zl{2ak1e2t1bjsbzE3AQ;af8E3&1z6IF8l4%IM{}<{xjA(m2fx zj^Qq2)`v%tU>o<`Vq)2*f4{GOPbC838RvJ5%2nf?*lhTQ98>G3lRha))hRV-1Wv4_ zqnB2d9QOkYI7w8KF8wsC_@tDI$dbmhc-}dZ)G>xhdE(!|+A;si-|Sv8aCBE;K@2!P zc)DPVaF^?|m}-M8wwo8x6c)u@5B%UYD=>McZeMfVXKKe}8hV&U`=L$bHz%qq;T4II zZ{`<&h|{mZ+n5Z9zgkGW$Q*^6r9kSJfePSSAu`D;v=09zyl_~EeZCO;LK6F*Nh(E( z*o(}+mV(${Q-?n=%UOjZZtPyr+v$Js91453gQj8p62CeEQjITp!px^L^qw1(0}^1a#AgE z-w5P+Uwp!Ebb^M7)09~jxVZ10lnc*nommbKr0FmrF`PWW}hN?#|; z6O*KhN!gp-9moOz)Tu)pNi~;CrB(v8*gmI95WQ?O_b+4H_a5vP_g<8-Xb~XzhHSwu zZtwD}p6m(TAZJP-{wpp25D{Q!w2>Vggr3i$p1R$91{L|%WOfcFG4PE03w%2ps*H~} zkh=ezrM4My15-p{9tkoN7UH#7$bkl5$+jf*}EienIeO@ys`N zD#3vb)YtvEzNaI2f;tbn#ZN^Wjfa=*u^piTqp3Ux2h{1;cow?48)E^o1Vmg3RXNnlSl{GLKe0|! zo=CnUHW>oc8Q+T@bjH7u^8NbcIPNimjKX5B)a$saN5f=<(NEC+As|(lYs7hS`wHv) z(H3@BLVtnosLu8)E!Pd@tsTms?e=f&c@H^Ur9+II}EIG>g=CfDKC&q(p- zms&sR7rll1PDBtwk98cV9n&LJNHKJRh@pNoW?@H7?Vk>$woPi-2?t=r-DOkF@G`Au=up)dV?z@BJ zh4&!5;>cVc9`D5~jKlKEr(R)5o;j!(Bm9~8xg|I zhC^m*e;N+#fq<)&b2{h_+0UraAk&~1y?Ce>m8lWD2rP3|kC3~|Xzq()=_TUe^lUh4 zgWiUD%1-lHWNV{xR_w_^Lo;oAC6r5eAV17v`i9^EZ~Lkq!QRC$n!z#9>3W(KV@1&* z2pq$b?-nHiu)X=>V$Uz5Zkb+C!J+k-@(dm+%hteVvzAbH@MPT6o(*t&0^Ex19tm(!D2&bzb6N_V-&*ub~gMK z*`a^3oK-zMEwGNkYpVI!=+ELYS-TnCm>=3II)xBBhR3HF)1)4HOm-DNXMTu$fO37P z&Z;791;Nu%_!TIemf35O2SY04LmiPsqZkoo4gd=Iee@B&|5NfKmyE-9KF#Plz)`=L zafQv!a(+q?sxWnV)R$2XA`C>cAUb-K6G0f7RG3>H_kqnoXpt+XnFDDVr02(G=M53y z)IvvjlS$C=z z9Hh=ej@=c6io||g4|;1}pWnF$9(EJsp+nzzg>H_nX9ZZI!-#kt?Dn~ER#X}ZlSwB> z2bTw;gI}tM?5;q86#(qaHh+Bm+D9IK_+kIYZ@%#U%^N<0voa7=IhJy^L+T9Pj#NRc zki$Few_)mrQaF$|>iq!?N@y?glNWD0_|obX&7W12Zfb^y6;>r@++BO@K^(Zj4uC`C zSYyj(Gy$hUn*cMUQ|7A7kauksev{A;3ziy@&GH+P0gQX-0?-YLuN27ye0XUEuauch z;&`=>vqT7=?dGqtz&5`$I7i!Q-bT$hdJVW|XG?n^@@yyzm()))e<2Dpu(&sPDyu4n z3;ItJ+`b%Dg4?HGDBKR+JAjQon0YOkN{Dp>n>ST;7XUKZM$PZD8~p3dex+kK)Z3K!no#0a z-;)`gnZ8j_arXwj8c$afPX{(AvG{dB1w>r{e6{&JYZ926I)#{Gq=Nuf5 zR@9MU#H)%deb|WH&9|*rRACKo@zX*USYC#TzdqE+z0f7a@zJ;DE`+F`dDWGc;>H-^2HzD)!AhcJ3_5LS^jzT$aWn{!e9}@WPpRsbLP9krDEqPk`*!g zKEv>hNQ*HD99INqHah+9?ewA7mc>I*wSIshJietl-s84MiN_+&^-I_!UlJQ)R#%l6 zZK?0bLc9G6`ciG|uP56&iE}(YBy&?B=WbCKBJN4AWVmL z;N;KZ{?!?tC(*nPbJiH{9KO6(91-s}7ZtF_upN_dCJPBQ-ysyRFq$_?LXGK+OAf~q z{*%o5kK=LW9TbT+3>dQ!}e~7bw-QK7>GhKN%buTaFJ|51qMrYXS zG5S;DN%z}3?~CoG?|<3(8s~G?Wj~~MncP8#((jDX<=Kwu#*Sm*J9U23Z9bCfU*j0@ z)bdi$w<9$U>K-r$6@M#mNAGfg05yMB z+b+CJ_n)AqZsmOlt%Bv9{b8E&ym*=J-&vl;FCgzd6dX$3`qXGXloiZ^jyPOiY|Vr) zhJyNL8zh*~ybs?GW(6T@MbmBU1%uIhA~uaJ?Hp`nG))X|zx)~vlx7n)wu?oXg*f!* zv1#?^w>~{A`ZJ`cWhg=Eur{dDEhCDvG58HR%Z2^sH>C+q%25fs(QnlK<{upVZ1dB0 zzD?!NffcFz8SxKxaQBWK%~)b})=J9+c=(5^4p3t$9L;Ur^!h>8&?Y`R4f21fpdNrnPGFz9t8gn%GT@Lug-(EM!o-ZY2J4Z~;jZ6V#TTJwy zz%)7Ad<*hFd~0qRs2b04LiNd|-Jwbl9p<4U){oug32g9>-BpU9bVNJe*)&1@z6>LK z)cx>NCy+2v)mxTabNUiP#Y{>>GbAr9s+~^GV&}$=A`UpA12PYnLsh$y+Vy>CN#{UVui*||zuGG^Yit;);g<52LjyA(@5*1lrpY9aq|7~S)yg`LJCkVj(t-X z-*2!3?k!jeq@$JgJ|nGY6}P%i5PPk`ehgyyfLa{$joaxeH*YLL&kmMk)j(&ChlFi?oGd zplW0%0%@e&J)WZ**`yj-*=@oR@fX;%O$x<$ol81~cAd?*HJlsbA z_H%_!Zobv)3Y`e!eD3B6{W8Q20`<4+V|mm^Q&cY&mt5Z0tnzO6vb^9>mKPkX%Y(r| z71jawYgS?8zoy06@GLI_hH@wmy>`fE81{*Lv0ohDco)>;T`Ry)$78%} z8~%C5yZ!LdNPE$31zwDI5Q9n{ILADBtJ@*i0sgU3HkbB3pTpkges7yid}yom#YO;) z={9bEahI&2Bm!4*UOSDy=$(`tg{i`J^V+X!ijc*>|C9;2tJOBthx;gg!0g(7hyKsz zfQw;Qs|C){PttGY^?)1(Egf2eE`crIUZUGE$9S6yc+LA+xiSRkMG5D8f;tOp6(cz{ z5*!Z-p51)N`>n4=)-T(-q%Zj2vL~~vZ2&i9Ze{Y&%YX6`D=&slYmWQAby7U6=g=HgJA-C@b`a%r!h~Hju8IK3T2#p0%Sw)_)+;>^LL>=YtvX4&fW6e>l&JSvRdI ztO!0F)<-T_-{HQvXAhV8y)$4#t(=q)3X^ULj6jARb_FYZ3-=bxzwO!i>hGl4k20fdOjCe55aCw^1sPl=Tt zjW3RA4%)}CgK8b?@q4i#WT*pm@cYb0VhPB{#a2j!*dFx1YIG5q8|^JJ?}mebHi%fZ zuh*!(v3rn4bt}dS_x5dQTl8j7{MXUo*Mi?7Ho-Sj92mO>B{(rw=&&E8;BM~QDY(0t z<)`A#6M23)YGiNhEfD14!tJrIg_?f{9-#Kb6@1_Fb0uP~WfFRO#V6t63&-0iPF=uH zLmwEim43K#vyD1n2NvZ7eaOCvLUaCDG&Bn9Mi`bHuF6dUlnhI-){ZSM7#q5G{}9yz zt@QoGuVAZU_t&O&|55c<*D>LHo_2#c?}njAS=_z_bOBO88K&5x2!!n+vHNC;;wTN8 zD-CGf=~1tDgU}C|--XAAby=X7aZPwU-2)HL#69r7b^hgtbh}E>5-%&3VZ@A7 zfMkIFLYCAkz}E3*l13T(cbWmf$_(g3SJSwh40LhmGTBYu+J;vg2csu{6$f#6^?aH6 ztqQ339b-2)oRypgAI~tW{Yr-(x_5)!lQ>@kqxe6|!;@FRa2OsPh<+1GeisL#i?K>^Ng!IC zUmm`wz=+QA;)XjHte{op(aEdLA!tG*7rj3-;>!YNs@UaT6Lg6yz`KGlbYeyVDmldn zpOB`v`;>^l;&gzUlahP%Lt4Cf>awa7+&HrwMs%Sk%qZh@!=no?kDOATw(wsoi+Or- zY2!zB9?Hd1B4zu@UQC6slx3HDYI*p?G;m#Il?&0@4d@bnGWRrls^Tb)C4PYRYv%KrBaqJ+YX&D$iy|7o4i2cmiYV3!;oaIj z;V}9SO!b4Wa4B2{|ACC3@f8|#;pDo{H}%%FpG%=&9Un8Fd&jtJopoADR&%J%y0cDa zou#tkpo5<5xGy3t-&@e`LWs#uVyC%vFhmn1EUnqGUz+wF52t>%hhs*0uDM(8zv@fB zL6EfEZMuw6W=12a4P_`WgiZo zV!>^5dH58&aKC*8Sp^PW2e8r&cNF;s7wEWr|0-IAI}nU8w_|MxU#jV zw`BeBX{_ic>4(?mYX#A>?0wKlDn@S|mCz2cklsy&7)uCatC2IIGyb?eMQ3#AgV43m zc?s>Y$Q*(iL<6MVJ)OQ?vL{7nypZKPbVdj*>@v4>#v=2VV0d6n>5S1dKrdGbw7QsG zqI!HNoxyAtnkF7Q4ssZnsM9NKfs*Ad4={22L>du6x%ro=4bgpkQ9akW>$A4y}dVVQeyqj1~f8Rj>oTHNIfp_^%p zF7YhG#NEYh;U~fn&t@`0fBCi6Gi~#mT!VwB(MR^1Y0*>+kTjtXk zOh8M#z)0ecFmBeoa5yeIdCbWGp5zDKfN>>TY&?+-#5N=k-{2=qpYjvPd--ED99&eX z6>*4kw&C9=f?~~`6UYg2;#V(-&%@*{xQ&WCwj|?rWVG{i+5RQDqz!& zA5FjJk!fH;I2-d7K~s_W{0UkdiZ)z%otk&@*<1^7V7z8cYh0cEMzwc>aa4@XGS@RwHzqsy zdzv{KV z_lul$k0P@dn%RRv#lDsgg9DKkN@8F0AzUMj-&?R2&~7Ut-^A8w@;>-G)ryfAAPlwt z7vfg$+}`4+-xgayr)h5``x}M2SN}fIR3$MAAu+bEwHnxTmXa9xXo-O-BqrAdk)7<1 zLd!bC_1(xDlI! z7!fSKv|60@{x49M`|T$ZjVb4~lF-HfiPO;p4Xyh3lUA!99*)uGMbhIwwMwV^Cucd7 zT*5jBYs_uf$|PI#`kR-2#Ma9kTp)i1R(W6E$v|+>pL&~1QEcQO{I~x{n#cFq z%ZHdx^+zim{5j_AZb-I(K`8v03)Kf3u)w0iQs#wZKw(d5U#kOtws(~y-fg{N2&o?@ zz%7Fxq}+D1r&#=(lb0q03aI@_BhvAew`Lf7@_5Q)`y4A{1C-KwAeY)1TiC6>J;vB= z^r+v%*h2Fy$Y=odW6^d0B-vy=%35SOKKRe)BF^FuNR->ntvD8Et>eHPw%wf8o2{WO z=i=$2HEuWmEH?5q4HWuutPeq)`!F68XjqVNLPQS8j5WfUyyLp~j+{)Xb*DK+)jEl* zN2qnD`Bt`7YsZaJ>rQjJvdm-QKppkMp{dmNrQowSsjjDarUv)8@QGb=0FK%Iq0H^v zyKp3YrqkLZ20X_oPw2@@jT{o@L^H!>egk{Bw4oT& z_wD8h@!sno)A(D7T{JvN{-*YC_wQdaeX=~pTni$Z+kbJ?psKGWl@nbE zc(h7y!J9l@q{5emzgWZGporiDaGFJ&OoZVhNd*5n(-MYHU#ms%m8YiT!Osg%DtsyO z)6xD8bKfbdMyErTFI(Rc@sT=2&(M`yf>?LKm#x2B`%~!cxZ&G)a=PxIyge#?9C!IM zzF7IG_K^+t&UZ^*rt(Wl9MtJVNm2wwkga|1WVm8Kz$_N=)bIa)JO!UlWd2{k7uH|7 zw#?ms9kg>{_g^mNYc+7;uI-Pao^kaF=ek?Qq8E#i=+<53(Fwb8an)`e?$hU!#5{dR zCPd-{3aXR)$RVkiyoTj;LRN&|pr39xPm6!bx#d%+l>$DVcMzwAy4`jeZ^!-0SR7#e zIKju2Z6i7bBL;4-;*=*sXAGrSO2Y)a$OzL)i#N158!-8X2jaLV65J|NC>T(g2&c-< zc#4bv++P{>!w|d`rtl5E))JXUs3f6x?-=X2!##LdfVD4BJrA&A+-5aanWwA$=h&Hd z1~!weKM2;2R|3%OI>m18Euy^*v*(&q)ue_SMk<{u%>OukuU~FM_Kv{UdhGaX>dqPzx+^a)f#YsYa7t*EdKzJ9s#Ua`Qd(KJWl3i z;D^GSlGc!~Y_GsWwnn4n-Y4%mpFpx3W;AmTjP9jezksQ-!~DvN77TQS41*paLG_v_ zQ~JY;V$8cx04Hi-FYF!&8>r97H=_7$IXYA)iBkKDFs77O!txB32Y(LX%HFfWl{u}I z@TLOB30D?|E6>AO=T;tdMlDh7#tXwOn<^dY4e)p}IY|5Sb<=f!lhb*a>P=YLxln>e zFMmxny0Tk`r;djLFPL~6OIXmwYZ@o$#w3NFvWa}A(Cij74e)cIRGIX5{`L;|J`)-g8aY^#c z!Y3>?uOzlqf#9(IvJ!UmqkV7Uch$FvWf`#_U|Yg15aTcyP~X^vJ@uUHdU0eKdwNg}omuPk=$rVnZUu-gYLp+$#r(2yI_FN5LMxryfWk9CWl#la9Xgrr$? zee9Y<4aw4h`q#EtN#b+*`a$pM`p1QQspTE5zjnd!o|^th{FSc0wsiTU{%mfW+yaN| z3*IOR2Lr5%I)0P-O15EjO#a4F~#j+e^H_=TV4Sld(8R| z$a>Z}6{`c&(mzFZInfDq5Y75XUg=wK={{!5(w zN*|>1M=~{%f4j{Wljo1t5d2@F|L*mF&iP8}{P4@hQ}xd;TV6{3RDHBtf25>8=672% zDWXZ$=RP-nrG6g%Q7l^ydJq*+bFhNP`5gk1Dy04{+XnxO@znOGheuZq6r_F+KuC+% zB5kQnK&tTd-;SrYKeha0eOLXLXisYU*WQWcy=E6vK?pH<*YIjXl}NwrVks z#W2sZ2k;JxgflInXnr%p<`u?eysCmmKjlsgO2u-qRS~z=!~8%GH=iKLZ5=;^e%Uzp zns#2s;dbt!FS2#yl2Y z0Gr=FtoJxvjtsZvnqvTH3@6^e$C#S=MjhY?RdZlt2KK2S@#9`e-Z|W}W^#Vhp&p@u zPw@bUXU%%JDV&lY?B`k2(R3)Ao_JwH!DO>gytkrcyrs_Xe)C6IbcbIv1%n4LL@ys= zZp3=qI!stxt9ziJ%kzWh`dSWzVC-{0j`WGdexU;%;xu<5%jQY>z{(^zfgA!UBePm} zBevl1VrE|eFEAbA7MMh3Vne}{;mZDKVzcGq~5#vA~G499yIQTyq zKV&UI^n1$I1Kq{##5Pa*4+ue}!zLv6XWSBly5L_?pF2$1Uy1v8#Que}-~7Ja!#+g6 z!4AjNO+nf_zKaMf{_vuVP#wLIV8%etD45M}WR1f;E%yK<5HwG7n$H3(rUDK3fXk8m zi@c$85~Mq2s3Khhn^Nw8KI_l-wO%o#WrJtgB$N@Wz{3d7!TTW@u~I}NC%eBne8MBZ*b5CJE5QWB)3O>EF`|ox3fG(v1duz8zL>wRC!9#=DeDl^SHX#EyARM3gw zvGtBdPy})Sz@l;-Nr|Xh0=V6XwGxC$N!3{B1##Nw4Hj6a{e~2d4v0KU>=J=;1eDl1 zP+CwVWY-9HY|+U>&1ZVTq?qMvy?H32L(KWi8y>NF0;J1?A%IbvN)9i7E7O-(sjBoKa-PeRA`8BX8T*r`Y~0;bjf%O>DTW`*(Rq6{3L zhoHvSd%lOS)^$GNM+;Um6MB3V!a#Ut0i5B@Ex=`PIK@VRC3}2}@C}LRM90zAzi*zH zpKCNfm!U2#Mc={4_$)=!`;7=(@ox4%<3&!*7&t$x7~`^B^HHqsl}A1$%`QRXCxK6~ zO&GozfP}2l*4fzZv)QsdAvLyu?8#QarNp(8kpg=^j$d?*;qp>t7gEXnPJ)fi8yZ*?wPH$Pu) zmN%kH&yeh|pb)OT2(lQx%}WGVBBk2!q^^1@ZJm{me{@9N zpH@?FgEY8(1iV@kWFo0?FPeQ-sIZ|D~ge%x}d2U=inaa3XN zME^Rf5b!Fq9VS^A#5x$n_azO4xZ(S_~RcIU?1eL@E)$~V+SUN z1*?i!CyYi`1Yzta*!1Rzr?yAgKNPn?Hh5ZAph>9V?C+A>L?p}5j`8F33yIr>bCZPx zt;8j(MOL8oSOyn}ydj5>kOb9R^i-RR!w8wdKJ+sb>Sf$906l8oC_wHAcyR5f zrwj<(urb9S(XY3CD*l7!o=@F4#UD`>UZ(qZfT#Vff@iJvrO|H<>_`JT)9SZpTipE~g2Alskj6E?pmL7u#i^t$Tv)U~G4scx#Cw9+|X8HXi>&JRRyP?9T zRw8kFap+dBO%HbaD1G#KL^&_~309>93bNc#JeCXq$Msm!Q}_o99IeKi2`hnoIgm0_Y}Yjw=Q{0Jk5u>_5Uy~1Sya7R|Vk*h9)ewKCPR9=2&MQ zi8EQ?IwrnCN5j{BSU=ME3VvC9aT?$SIjb%cOc`UfgXcWKKwpPH4%J_$pnjgB9h=6% zb9`9t1@H$rY@j>`%kuM>$m2{ehkG5*7nds1uU9zxb_6UD z@w{j;Km{%;NT_*>B$nvg+vRO_;;p9z;~M*R8NOiIE}j>}u_xWP-?QFTsdwz#pIYw% z>Yem$75nyjBJyhV?F%#p0)!0+rEj+?TFSe7Y4^S@hTUr1DyC$|zG8$=en7Gvt@rMh zs>AL%Oewv$Q+1h(+sdOxFY{xNJ30)e5z!{pFrCvI>En6-k<(8(ykJ5WTjO~H_=XC+ zwT6gg@|;(nWa97QQ}Y>Y%?^n#S4pu(yc>9Prhinv--z!K_sS-e9nHBLuOfL5;9NBJ z4!I+E8P2RRB3Z+xA+KNDn*<_RyCUv7e3}r>%82KkX_w!aQhq?nKa7cR^H^7CKy%-C z=XvX9B9ZwR9-Z$ukBtZaVV54sIUCjb#XGYR8Qc+Z_d^~2a8@qh%0N*Z_qdDo_b3py z;hYb7IxRlzK1Q{zE2lTpVR8t&m;1Hb(H0QBI}0xr;Um`@%mvIL=Q|*(c-|TqopI7? zr{g_JM9Qctf1;GH@zy-X-!Ef3WBGgOYV864`&sgQC$M>?JeC=Z2$}2T+Uo3e`W(j3K605N+6t1%0d?8lHP7 z;}LfQgRuJ@8Ukh4LfCC0>lXJP@rJc-aj(TMs%Kl=8}u)+DjGMpvpG*O=a#GtROL#% zkK~MB!h!=?Uh5?^hn!*7OJwW~X=l1PQfO_L{2Pa}GYeHScMq0kW&_Zo zNQboZX;g1QS_$-pXvzIVFWOnC+c}Eyh&#d{?EVzT=C-)s!c(>FRs2$Idr$wez-8NB z$2)8c!8V$~@cWW;4f@xF*BwF(m713o=TqNK?u+6CkUkCM2?X*rHcEGf*j*i+5hW5a z?`9BoUxRA5xX0nCfV@zH)(lgSYa!Ho90_-YvN2^bP^OEo`!DQ+qK$Y5U}7b>JR$5p z5&4|W@gR=o_}CoIcbK`8`MgMtC-aR)*l;@E!Fu71uaDqPr@X)5c$!=`jjK4@jd0es z&TT-40fA>;f>U_91F5TesV4fM5XAF-E+?&?K8Q*A28vk_P;5N!CMiyBneoYd!e{yYVQt0o`FE_XnA;{$)s#11POcQ+dG?QBIIn?1xBffWoI5a}8L8_M#0;PLp2 zoqv@pS%^zsZ~&V`#f6sDAKM5s*TX+Y zeb4gd^@W{pYB$c*hxd3m9B~{EesygH4qVI{oe?_OKov&j`lTh6WpEnRe_dZl#95<_ z{7|O8fI;AT37R3O;4~Q%A;{4@BO7OkQTbpPLt*L6X@?XD{ast{R%dj6%(~d7AFI5* z%syE2>KtO;jyQJfl>!_TnT4a|z07flh*b-QKjeWfN}C8zmMgNc%6b~Nk=x|qd5?; zpmevnNRraPsoME(9gls0AC}$GeJe6A+bYvlPwB>1_sxi#+hCA2e~)Au1B1;~D$*PH z`W1e5zP}=|lS#_*HH|AmR;UG@WvkI`&4)5Pcl-uF%nSu-;j+CUIM>P<$O@jx5g-Zu zP{w~P@n8e6-3XRAb5Vv%6mOYRi~yN-#Gz0_d8erwT!!I`&Isg}az;?(8Q^}Br-aic z7?AS_zKPcYCVWUD19BcixY^x^XNGhiynL?1oX%y=qi;WrX zC-4J8|Cu28mJS7ZTxnAqVP2+xyH?9?Xf?4qN*?+}$(*)}ycoTuNnrET3 zZWqaWBt(g_2NRg$yxGVoirhA%`I9)kGHf^VkgF4}zu;ggXA-`f2f<{xn5!LI;IT`Q zUS3!p5nvWMysDHfsEA1cej*Ey&z_*8W zZ#RFA#Aq}2SZ+5z1buPf_?xQQX#P(edogDNFFVYoe3JfQCmr4Veg^Dwc%0DuNk*^_ zW)kEj4i%$dr0;Z_nFzQJ&v$bBAcP~h$yLGa5aPckb`mma{)P>Z!D_O@`Ua!BLGWO+GV|b`cg1cvnD*hikLGbPERwi9%bq|zTim#dM$|2{3W+d3 zzTCV7i{j>f^5X#f0@95cW0wC49v|XE=;;1EPLTIN&?|;eu|mOBhOZ_Rj#9xTJ zm!deqUF<6;K~3RkN99!8l6@F&#UAEc@lV`)iwwnkalzt12TpuM$MlU*$lOKvZpN{0 z;#+YLXentP_h+oH^a!1A4g?lVH$XIlc!W>>Bi@_m;H7WHPV*2DUHqbN#q%cil>%nZ z4>v|OcHXA|dK5@ibAOJ_qL1jIS!*OdieuU|!EK(=d5@yx+o3wdy3pq6_&A)bEGmP> zc_+{a;tKVFK3GJ8=^Px-`^}k^Wd7@>ihBYy&8XPj@>%Wke3^PtEa+ zPzee>PlP|Sy}RNtw)SHrU<5a!z&hj}qN)z|w?4%4E}R#O4bQS1+x`j0!pK9r7?f1rjsW6GvUA67Rz5sp=fZGv!Fak%4kKGf9)pQenB}GS&D9SpL~@>31t^fNuwq|P`zfE{Nd%?GkzM2 zUe>{nP>3sy@G@3IK;)%=?Om)@=8wx-@zXqqVCfv7mnu+EC8nbEgF2Pe6wiAJWzv@x zCOasP+v1~TU7kCE5KJ~$={>9zwfW+Cci<lycF&NuCRVel+isQ zutzuu7631ej;}R;!4!ZM%@5Yh^)LPBq?-Am(#VUEuHvoHNf~8*J3ROMm+qf4zotB#RZ!M<8%oY9jqKqB z7Zcrnp-Q7UrxE4Fhjm>N_l#@rnc)f+j;n(E>fmYP>KF`luZG}iKImOkV8HaJsx~|i ztZ%>t4eoOvfw>2ZrKMr_nfTe+3-36qm8fSpSyS=(~$cV_wdU zy#+ll>KPgpaX*4Q%^lfW-1p!I_5rxU?z`{{cVT;jgPJ?MTii<#MOWa%?f6Arr{7qp zt3~LiT+Bjcpg_MDw~uz$AqhLhXs*shPr0kuVUF`jmJWj5?;AiT!{syfZ%oPNBo`az z;N}jegv=cnY><7le8+l|rHNYQ(l4A5#M~F|P~=GbAx@KoZ$0_f$R_i5Z)j3s;|7q|r;QEZl)l3}G&Fxb=Myzx%E-A|Id-uhCqSkBk-h@w``0W_@LtRF>t&^Iks*!m!;q zm%pFF7yOP^cvbv<#>>2Tk!!5q=QWD)ySOaZC|+-@Lv{E=NoR;&2H`;6#cS6WasT0F zfb#UH``8q#UH64YHeWO06I#_yFbUiEY*HqrZl)NjC zkQZWfM|{{>z)XDDM`%HO*n9HaEzkY({DV9nljll#eqWxC%9HJm4=a=s2FmkndG?p* zDtW#lX|e!AeAoebW5{!xJa@?Rd3o-V=TnmYUitl9d1lM=7J1H<=Z*3#lIQ9294ybC z@;oRdyf4ou2XU|~K5V%>TjjYzp7a+OANC`8-X+g_ z^$$|EnB9ETEf0Q8k_ zN*SF2zT$6w`yj5V@gH#Eh~x{wL4MxuIHNM_ zLH2>tj7(3kuV)Ry=6zkTo&<_f&DQH<>)_%ZIu-R$SOsb^64imN8}+=QJaR1n+Al_; z7{G$c1Y1EMIw8xP3u=y(Wfjaiuv_3P`%1h24Y5sIK9yZAh#QO-?6%`H0>uU=;y%W+taya;Ay=%$ibq$-_h2F}<bNFuJ!|&8j_a9$}F#tM=-uVfc z_3q7?u2}86YTK6)S>XkIGT^k8^E{gs&yVG*xWIefr@FnmJq3xl;A5)Z<&OMpuaxIg z`E@)$)^2e0k@92j%vZaHWWUQ*erYh>pGUt@Wi-Fb;UnMd8l%q5no+3SGJ69Lq)l67 z-szJ)OojfhCp*ASn?BkOeaam0cZHdxEa}txnBG4!O%-X=SJat;nWGH*fX>H9oTVv$*-*lZ5Q+ zuxbzTthvY<5FJm zyLMdabHpY8V5`1};{T6s)9F)mi0Zd!iVxjo)oP`e>vjblz6U+R}r46yKQ)Dt^8e2N`Fgz=cT|~V8t!`VgP(U z5GnKFYIL93voEfvz_#A#RX}@%*Sw%V%`rg!1Rl9nm{(8W!^`>${CYgp{XaMn_woBj zh4y&KINFyV^O*vl^w*seYv; zm>k#q7H^IqFVDIyMgA-f9f$mR2Z<H z1iU?u0`D!5)JgJvYw$SWJx{~?Ze@CSv#v;ncWUvG;cfhf1HU(419-1~{K)v-Cc}?o zJQ{L5@U|ck@jK?~^zhEadnbMeHys(?j=wwL?Rx{@eQ5QO;T@U^@0^C?!0*dQM0j7j zEI6ro`OVs1V1y|ZFr@9S&8og|3??w?H7E1w9Af5dtXj`m-wbcT=Hin%J)fs;k%8C zrf=ce$P)sQibhD_IwNvbE$~rgdfy-oiK&D(B!xF4T;an%DQG2yJh^s=l(Z;t4< zrOb=sX)x-9zYFeF}ggXZO>d zTz*3)ZA9B>&i?=WpYaPUKH%>L9oFk`i4K3H!v}S^Mu(ep_=*lc(BUUK%zj9fdx{Q= zbm-IJ)jFJ|!`pPYLWlS2@L?UU(P6s|U)SLQ9rpjFs_!fvmg?|29nR9>LLIj1aHS4c z>u{3}U)SM&9cHgm<>%|LP>17nI8}!aKc>>H)nSJYzpcYpb=alDKEKg;(%~2#PSfFB z9k%N5UL8KB!wwz3uET>mv2!I)A1!bzDkE{bojasd;Ch}FVtbJ4wvij zVI4lN!xCMtSBIbK_xI}ZEV!TZcF3@B>}WJ{@k;VgJ=C|6v`k*Xi^1 zd#j#@^>-`&o=#`QFM3p!W5u7)-=EjvgF0;cwMw@+CI6-R{YV|&tHV_~tk>n=q~kqx ze4vhZ>G)rCxJid=b@-4Dm+SO(I^3NC$9+0oiwW2?4|}fVJ=`EkSgFG*9oFfvNr!em{p01aDdnXmIL^57x+-Ut zF7I^r4~C>>CH(ofWVyWh0~ef_j_-YVZ~ZZDq@=5M)w$-l&T-9iHQ>1jzvm;I;acFT zch%rm11^o7k9aM9&vl)P=P3LaxkkEXB0kqD+I2Hh*CS;O-W4NF>i6>bbAofK=gzsM zX2OE`Gx2-D{Hv?$=TzS`w?@jCGPkCtuG^;xQ>+TEslK_!KX-1o6xR33kF%tox^!s0 zZ+%DMuPG-(zE|mRm#O=wfAafwjql1S>bsTxrs|pA>iYWX#oqZfi@bAd=Fbk+vW}Je zR65`3icVaosqhb1srOyp^)tq%O40_KEXfr@+;XW$KS4n3SQ$g zh2MQT?$hxu9k0~!>_sa5G#$5!f~j8qtoV__&zsI{R`oUN`tn;;ysllr>(lW{9k12# zS{=Vj#~XEgt&STyzFWtublkN}x6g{Zj{bM7_T)!YdEVz09hOg>dXB%c{G6+6>Kni# z-r|v?E*M$tEgV%e>inXTq6>$+`uGaZ815~&vS#5NrYPv+;tJ-m{*DEQmH+dm+x0UA z|89-{eLB8N$GdcVpN?m*ROw9}_v*Oou;l<5fD|q2rA@-mc?qI-ae| zTczU%^>?eCM+_ItubDc2{21?)%CZUN6U)I9<(FUPy}InmDF}O?TXbF@D~0v0H~OY{ zWYNeX`%C#{Q_HTL=$}xQ$T#ZTlEimZF#W12Wz$Q`#$Pq5clY=HDO1X>oQe-F*S?=C zd=+K7MIN;4cwt=OEBAg)KRSL;f6vwJKi>XK#m8=|+y_+oso$-*m9A>l(bHM!zigP= z9*${8`%6uKEM7@Xf3&js?ReAwze;x`_%RNQ8PiZVXa4kgpy$!VsLUU(&=Pw9)Z>sUm4{-pguCJMY^b`t$84FNEWALbDG3COV z8Nmhh7frp`8(gr!J9okS*((3cns0_`y1#Z6zNP8^mwG(?N%DAGl@e!Zq#5ihksUkK zj;Js3o%NpYtoKZ3{X5@h>NI?3{bo5<8fVD&NBZ}^so-y$t>A9gVP&m~d*4ytcU`68 z`*c`&wTkDzt-kxhDqgu<=WA7=N#OA3)9H4#srUQtQQ=A*HtDcVhg}-4ZIe_wOP{gA zNj1TW1v5i)YmDmoGw0Tn(Wt{e`~Ac@b89ZEo>!w@yK>)Ea8?dg^7a(ev)o<+pp1)0-OKRRfdX*Tz)6D|NlO z8vd*1-#mZ8qWRvM#u+tr)M|BmMPH=GR;^x+@v0BW@4a*8d+X{kk}n7~co)=r!6h?7 z^w>lWCnO7Io#ksC(sHs0^jwsQ=8mOW-EvOHAYpWZ)V9a@Q=2!D&uyzhgsjI2i zbu#aw>N&x};oe0^Nxp?PITtCRSOjZpybZzn(2QWH9%^H-y5Z*Ov+Eax>Qs$Wf;DyC zd7%bj^|}QO(AXEUFU#l8tZDQv2nD?hW_jz$lnt!90gAo1zIy&_tH1pUd~mU#21(}3 ztD9Rh4^0#3yfbTNVVmGgtx9|6&Y3qS*ue4^LSvoDrqzP|p>@xyrbJLsxm-})R6{_)$8|RfD7Cf`(F1V?B?sST``UQ}O^Qs%CQ&V3AVCt&t ziSD^`8s-70`C4ukjbz?=i>FID7t^|lKgi#k=FFctebn^xMKe#ti=1(-wYywUA%Ge1 zm;kQ`gR?6jq5RD*koFHLL z`CgAsT2Nn;NRje=c1_U!hACge&8n7eVy_%mKl~1=D#CAaJIlhe< zGe0!1dO8G-VsFR$nKOUZ0-baI0+12}EYbqk zIz6N&q$8we!yHWARnZNq-7`j76&|yRg>xEgDs~X~Oz)Xzg4t${83W>(St!K`6w`02 znLTIz*-*^TBK4+b{>-z1DqZDp?`h*wz7DrLLjUN7A9m&R;Nm*n4DaW`ke&DVF*IYO zor$vc=tOWd3>D4;&Cd}^6!7tDX3Y%rUBmR6`TRYz&{5ZL(o_PeHVh3GQXOwJQtdZ_ zRy)SxXAZaV+V;NE8&+SS^ohC>6;|r^mOj<))!#8@%=2F4oq6tt38*H8s!6lKc%P?~8JI={ z0mJ`h?p@&HtjhfHGfA3G(p%D&k`yRYC=f2K(@>=%>huOF6d2P00jrbSq=}@-IGNDk zZ6oU?BDxV!tBY>dpO?rAM!d!Qh`8HTbR(ihMV!^DbwQkkDiOi`f4}E4^Stl86Uyp- zKL69t+h?A0zRz=>^PKy6&rIxqtJ28>Log5nd;)V+_w|YX-UQbbl=ch;U3gLCbo96oCC@m<#Lml!&2?eg1$L99QM!PwT{d!t)K zV9HV$3-!XqjNV+or1e}feB)p@9o}m{ysAH=4xrb#3A}q~gnL5O%O(ljaHc=gjT6zn zWT#$17M7N}){s7u!a{dguP^jDDn3CU)+h#W+2!%X#aFf_+TyKOUcB2*E=VfG?`YYz z<1)@$C`O_O69Klarg3efhJg5Mit#3=1lN-ZE;F!xK}W&-WaVu7$Y211TAY5As<;S{ z*;_q9>DmmNZoMWXjosdON(9C0ajZudcY$=7U#fFt*f`keLk*1R>ay}WJu%#kUf!dN zY2yt}CK!2pD!|pkAnF~a1jfSzPdpaZm_>93g?Oz1^^lR}f&N0gEC=I-l7fG*;mHi` zM-wb~)adueUt{g@`gvR8hVxCSoxR@lYf94f5}jQJM;QqW#hpAK!6c2z#dHUo56(n* zW{*8wCp$&K-X4gU2xBKZSHX0F6Y<6kR3U>JV3Uf4Iu}8%K1Doc<)VEiUhjDW@H##KkFCJeAabA?>x#IM?Ww zdW(K_{mAb>Z#=5O96{()1=){9eQ3XL^OUcc&$c~5{w>y*S5CwS%6aXPy{<)wwuX#tQZTs?m|vNpGbGo z&cE8u!l6KCcQ;lXg{=Md`f*yBCm-Ei4`9WbzL95R2eELl!-d!00GzYTqmT0@rU0|{ zkoBtVHr?MuTmAi5C7T7b(F5Kyn!9pyo1f`FfJ1}M17=_~cr+sp7tUzt_QX3Q3-Nk2 zXW(}3Gk6mZcwt7A@xvosIJ`4SCSLapahmAzE%sDfOlwauKGBQ8yI_xw7xm)i=YC9^ zX6`om$#|?$F(sfdh zyJWoV|0W)-EA0-XC7u;zc)bo0t3#KcCtfQyc(WhDDIDt4ENX(|U3n$mvwX4n%XFN$ z7MC#r>7M0GK)PpHW7F+efG%m5%ij2dX-p;3&lW$D!a7l|e|c7#OrY0VdBekTmLKYR zRf2O9*T+&H*TF&Yrhk@>>oQ)}XDXAhLxYnse)q>azl(9fm}BiMj0=``{jfMl8NDtZ z{njHd;!Xc6(G!d>^j9CAP0t+9d-E;CC-ACM2a~Rso6ZHn@y2a7AL2dh2k_FbTHjD_ zuWR?3H%A&qOz26N_!9j#KB3iEJlY4Ap8Dkt4p+^LAj{!NFFGF(Q%;ed#Q79A?luIZ zV~MR_^=O3kqh7LjcHJKNZkdkFqieM;|H3@x^M7$Za|tFtF(23pD~azN)FL9D)03-S zTW+C8jKMTr$m!WpjL|`QebOFz=37j+CK&y!e;n7rD$BcZoa41Qh~VD4V2^+N7Z`8* zaeC~0AaI`cB*%X-KNQ_$?egHX53PUFY*EP!_j~D^%mEe>Q7(} z59ffMXpdvW1iy`7$I8eh;;LBkWKo^$If#zNrpry3fM7(WST(wOmh^F_b;Rm$AX zQ~Lc*|KLc{Mysxo-d?Oo_QX^g-_m>LNxF5wWTLL#DtcNkH3ICZf3E-G=Qwk?}g+=x;9AiERf`BkmOk) z$IBu|4R&jNJ~_2^f@*a2hC*M|bi)h1r_q|Dm{{5~?;SVA95KnQ{T zq8Hb$uvpcgjOxNa_Jaf1>JAT$q*24>^_(8VP5LH`r2}fe9{V^3_ThUlCb;xoqz~bH zF)p;J?f4f_d-Zo>lK2+P2oU`4iwTKD>*aeASkn)>8&fhK`=sHdC&Apq(RZqFJIdax zf=E9&Hz!ep!Epy@*7|q}uMhL_UQ90@aLKInr8wlK{%FQ!|2V6zv*Y8FCSC0_e8kcT zOL3J;zd>}TDd;@ixG=1;R&HX_$W8yVp(#t_md0HCyve8IW`oPW%}{0eI!j}g9&-5i zTb!k?9QpT|c*m!#J!5Y+@zHl0nzDTCPqv)ao@pDOv((k=HM%XBZ!@`BGKp5=7UWZZ zHhN(ul(!HSuFsPu9k}ay(bsZuQhXV)Q;&`I%>Iy*%cX zKl-$Z&)N9s?@jtvi_hl`Kkr@)bK2OkGza5&OBf#a3zh5&uU zJy=!Dm-=$xBcKdY-=x4%-&7e~hY#0R1{d++ngVdN=Wu1wzD3~Yz{fxtoPCBX{?GVv z;377Pa_;+y?Lr^R(IIdP>5B-Q)OSlo(LTx_0v`coaQaB&0;lT@yq?f44lN+zWF-9# z0(nQ>_c|P9x%e}plZ&f#ezTyoZyr?YCytiJ5$^&&rOM#$ulC~d;PnCkGOk=K$031Z z{1o^ID1*fH2^{4{1y1@i=EJoF;Iwk_AM=SECoX)PxV-2u&OVW){>XgijuV#+z)^l? zUD40vv*0734CGmFni2moenQ}!eT<(!rKpeXTM3-gM>^xfWx+>?M{qd975``an83Mw z+5QfnY}(&ED1RF_0$dK1VIh6vz(+L7d}qbA=j=&FUmi4!^ayY{i*x!&$C2&=`3dk5 z8{u&D!zb1japVt!_v45k6FBnY0XRf^0&z_O=lsI>DxbbOfnz$$OFFB2Cq5vMp93Eu zgWz!NH@6EM%P|#zBYum(k&l(ZjWrbgLfkm`2q*)2XWv67dT~d=M+icY<(={2vI6Jy z8L8s`j2{;`sjp4wV|hCS&gr9H>I9B_BmhVJku~0OlG%a?W^J)vkH_05BifHKH@vjRu?!vQ!gB>p3AG62VX`-DE`I~sr!s`N+d zYYMm`~rhz`1;B&sKqBd0Pa| zO(llh(}aN0hBbNMpI?6P9MoCkB@ zW1tLDUxUE091-vlf)E^z`8>F^sE_CKHPK}Tt?t1Hzsg0-)q#%?gdpv^qqeAza#P?V1R*$_;fntgH+G!3jsP6xx6~B#rJrNqBcKcm zgy9YmoI6Pz_Gm1 z037u_6e;Q>p9k;nzl^^{;Aqbz_!uaI%r_=*jBf!SAqYX@8U&8|BLe5jOZ=m!dhMGB zA3;2W)5mq^%;{!)ISNYox(&hmo<5|zz)$1ggZ4*#r}=Q;{Wzz`QQ9+oR?$9J4wvpI zafgl*H+G!3zT?Ejj}y1`IB}8V#D$L&mp}9P{5N-;xa@J_rj8RAJEPP-*5~B8#rDGX zJOw@i%D{BD&S~@EI?CW0e7Hy%T;Dmxe3?%Qd<2xiwLivRByhB^Mc|x$jL)52)JOR| z_y{P2(?{I64|ho5oIcXoCa=CZ@Dao_IDN$3B5)k94hfvo$9#7R95uHG;1KNzl#d9U z%=b~WOBbkrUbIt}FX?B_FXG5&!AC$D9FFqmA)fq2pyc~NW1tKUXSm}3jL$X~^-23C z5bpx@9TGUFkF?E)>kv4nkMdOlM?W_RoYXhFxtK5G=fFom86>XHhf4{Z%a{1AK3uE} zZnX~=5jdG|?s-M~SdOFMBcKctH|fJomBDrRaDCuoTANp2#D{Atqi<$Y(LUOf1s}2c zq+bsCaFbM2@ImFx`f!KK;12n4lVxzD zKHOLt+#VmUy$mkm!!?z`J$;_Hyx}sqJp#x6*ACvlj$r$!^Whr6JO44=%^#6yF<-X( zCh##(28rwN;rhTwG|G#c^x>w;;Kp|q^JO_Ezz6AT^Wi!IaMp|hq1}@^i~3yuC;;pS z;>H7TjNgfP7wDHb_`rRN4_7DjQIGTU^bW7S8SoLCfy2=cJpxCG(EywkD-gy!~^uP z9QR#Z%$J|$z(+tCoPCBX{?GVHfphjT{`M=2`lx>rd<2xi=_8H!a7|@!N3SUAqg)<* z1e8JM`(=S+zJ~*FjPDUR%8v@1v~PpJQ7$TQE?-1@g0}DSVEaC^r&tc^%Yu(sX_v1@ z85qV-2%O88@jb0YeQXyg@c#2K#zzE>a$CViKpDuhooYt>$9N@hE??Sx$IHC=PJ@pS zkKpu?=Ho>i<>$diKpC7q;&uug`MAK5cl{{h!!?z`&53qJ`COTHwe#AdeJocTd<2xi z#BcKfASs%1#K;WoTyXZ0 zP8H=_u4D1KW&2^R5FVDG7Mm*E^fbv{<6f_6Q!1!|GP_8pr z-_spN-+4%10{ITm94Ny=`m&IVXjE|=qJ33X-xg5X*8!TdxP|l`PL$Dyc*<=74O1Ve zUmw%&e`T?~ke>q|R3C=~j_qm^d<>Mq*+<&$!*!IwRrzpr;3HIqAob0@qG%uc{T%oR zK?n}V_#T0yeWL+5;v+s>6Zjb78Js@S{Jvto)K{5<#wD1*~S|L+s_QU56TpnA)Te2JSE`MUBl{=)*t z_?ZA4^Z!hrx4c>KLHTBUxUn+0Dj%*6d<6M0xbl+DqTXE~KNrvr7~gVz5yx`H!AC$D znC{jGUw%~)$4|51BcKctH{ipKf%lh}`7HI}B4zYt`+fO>k1$gNm#^W9|1(34P?p18`1{qr`Ot;23|APhU#_&gpTKa*+TWMcb>p8 zek*wYyukP)14Vsozj^QxPzI^*c7bC#Cd=U3e7KGP9COTzd64l(fCj^f9&IHVt zi1q}^jSHOYmy>93E>Pc;Xn(T2O+H+-3~uJtUVT~c5u1V3*W=TdDx)v&(>Gs6-##C% zF964Wv{c||X~c)4se7vL!#Bmv@6KNb`sg#2>e1iQJH<5Kata1L8_w}^F5NSmO6Za1_>1s|MnkdLdMRlrFHi2f=qhs zi|W4Sa?c*EQmKdGXFTNsi!m*ZM;h|m`q&squIf``rO(F^LeAX#H zFQ!NEdG1%0Y8p>apiZ_!25ATLkwMx~_gkFNKrjAwk%V|YQ)fI^Os7mxKGYKwnBRnu zcNn5`LS`JF`+#c-z*1%w`MMy>H!XBX9LMm5aOLt!As^=>DWCG?qkMKc8QMS{#N%&B zk@bZ6Uj~=4Oc=~+hB31_7ZUknsOujjRp}h3N9AsKxKmL}8RINXcp}Y<&>v`H& z-f{5d>P#wS4x=KUx-$Me?6bF=FPv@6gSOJHywLCRmN@!|{=nZ3amfWLajZku7yfog zA>Dm&?ZDrE{OzE_=ck}HNu8|!un)_4+A|rDKP@^e@T@~;yR4@cfpPjs8R&cZ8GmEN zM2P>}9}U*0+#gi2*y-N{kG0Mg!bX2rr5YfIzeOii^8QC{kw zMmc7o2YDmgXwD#p5)(p>c{<(F-?T%9dDy&A`RE%Lq#pW9 zhPg8NZT02nY@+NK%G*vo!f#GzIh&%uFXRiUKMMVaQ2!kP`u+7O<>nw4hcEFrXq-6( zI*!0PF|$mz1b)>I@ohnU4(9WpELnAK6x%&9YwDeq?YM#)VOrv#<$&qentK zpP4c(?2bXk>5%#4e7dEbH2h775?Q7>tn3HG%P{i@+Sl*AaWb9dX&33aXTdYiDPa78 zdg?^{7|I@G-yx9~bvO(^&$BL&j=#-7))V|?AlAvs@>5=h=7c9&Szns;Lk>3OQAYgMpo5oCro{eGOdq%9AW+&jUPcG=iwb#}{|xg-8h&$; z>O4W>m?!n)Hw--(#}LbRW8ps*(;J=$o(EcdIy!uO81{_g@yE-&+-K%NefaH17geAv zn=6C#6U!?D>wxcr7(6(iT?^SnS@x?rp?eyiWm%55-^5WS7oknKNZw zpkyC#E=c{e&>w>y{1%%_F3^T!!R;{zybKx}h--_DaN^yhKJE#yP`BB*}o9~tNim#3^p`bUPKad$TW92aG%`*ZNv zb?6VJ`A~l5DX+ZK#n`RT0$AXc7go}%Tp)pbvmhoKBs^2+muqJ{w#gmx_9T^)$jVZ#Gj7MU2)HAKlPh? zxBTmR>e+GOzBc?#(S7*Mar`Xo#_s-o2ZnFxPHcYRrp+%%bdQd1-jaC1mc)S`{4neQ z{|G}*5`WjPcW58Lo0~VKdb(_`F9zik!E*|pOLnyFQFSRs z9SUZvHbtY|z5CJk!E@ifX-8|@hV~uXDAt0wXff_8#*KP$yh@_RJaN0V?6@~>2XPbL zI4yg~6Sqgp;@*U?Wsk;9dE(+4H|>qnxEW7eo5p1kr^~9#)~0c@h>OG1o6__N>H~Kb znqZ^hH?+}S-p%;%sf9TGbG*Da^9eK|i+jy$UenVB_o+i6^Bsmw=}h;A-P=q#JaN~V za>%$ggPZom8657E=zYO1*9!;*th)(WLpVjBOkG_ob`;xk}#yvT^!1q#Fs zrD}2dx209KN@d~5O-8-GqoUg<`3-R4lgRJkklNH`{%)(vRpak~1J6HagnCfX#cC2{ zQ+hbvy&e%g2f^Z<9! zHKt$s(;Z*G0=RTV^mA7McXy>;Ti%P$uUKo=nI8vbDA-e5rE0fStJ>!3l$L$K$j;wo zWZCB#)^GqpXj}#RZo++8*unm)c?>w@*{1z`Xmwbv?!nWxSgl4rtDEamA(3anjxv5~ zT!#FhpXHf_O+Q<2%27_P@pQDOEz4B+)~eJDY?^`$+*$C$+SASFe*|KC2t#J%K9ovpeB zlxer0W!4g3043jeZdje%Q?DA@>QIgqDtwPAM-+P5sfnS!_ruxzeAU@T-zP!I*PmXi zPR}e+>w0QcV_OY$gxPL#^j{b{h@t$2=NSAKKpB>vYj~6Y3go}s2Ttaf7rrNcD;q1q9#E#c_M_YUJ*LgaY<}w6qRo$6IoUoMfIA1H1@$ztzoB2U ztaa0s>LwR2+U?O#R;s&PyufD>|3E%1SO1!Hl*-i>^Zp^S_X5%{41?p1wd)OfR3w1{W zY`97P)d8ix4ob{}pv*r9%KUY`-(<$0G3y6J7s}wuL>ZRN@s1M9z&!xtjx5HW0@Q^V zY-#~>vHh}mH+YV?gP_#;I#5ij>aC#E(YT~iE!n;}UX6CiHb^!bRi!x zPL=q|`OHUU`>y+9rTPJkqCX1q*2l749Q}f+%SqTab}5WQaP2bC1~<3)^FA44UfWW2 z;+7?tyO>7$++J@2ij z|Bi%J{$)yC(2Db+nkuE^wk+0s9B~=OK{eAQFJ(e2tJO-#)MCEld}{MdXI$GsQv586D4De~A61EV0XfcNNN%dlxO-j!934Jr7E;Xv@FwIw+bibb?K0m(!I0lzjXhpSJ(X&{sz#PA#c}dFfVep6}uiGSi1HUw$R;y0y zS*A`woj0J)Yg?=HRk;d1U(C-G$1v)@^jiL})R#cD-Lh?mLn^!-Ht4)XUcv1wwcg@t z+u(QDIr&PZ&cv^zw=H4a!q4kt>VmqC+Hcww`e4GXS&rWME z`y=evI!<%8*1#{Yw-tV2d&K+^S{YU=+m@jH)T))O%X3RpOF9a^MZ3`bqN5_Zo^9;d zQnd_o{Gzs66-i=`hiCOt(>^61T3w5|81r~*T^<@|YqR0h;*RQg{i+K3T+dlk*7jLz z`!F{cwAb|$OJ!Rc6Ll2nGv#~pGX>eTu(s?`}smSU}C z{FuVush-BOZ<+C<M=&@7LVtz5V1Z8H2wY zRO8R8RSk9>*?>0CaO>*)%3NKFzK8)@f96tkX3q+>9_zf5(VmyL*5<>x>e;HS>4!5u z{i8P+z3UE|d_D%seEweV^ptuRsLofcGbV(N+8d3Wx1KQmwql%gW6?T}AxqIWmZ%l! zCE8X_i!Qx{uIAnXZKVwq-BiFanmw&D(dZ|o-CdRf5sE0 zUC{i%xynDDx&FX-|l)U2I#(|La7d@jNzfa4%>z&Y@5!40;1bAX zQB@o?%mf5Sna)0e*-by#-s`a-T<={YMWH)RL4^F$q|Msd`t}jA= z#plMQm1^mBcpQ2-w$ayt?T_`wdBHcAHAWYy)3&TtE72CHgJVzrbH%nW20t_0cbh3! z=621h_kgmT{&pe9^~2EP((BjPs`Xn|s=Bndh0x|n;HZmr_wn}{{VzIX^fxZB{~Q%^ zl%o7BDTFh<|m~d0Ht3a24$TsrR38Fmes_ZsBlA53xue>o`g53Q+DYch+}8vHKCnzl-{W=n-ylf(R7rX5Fox^MZo(Q^!x;gUOTxdQw$ z|LLM%vj1XmzW`-;%<`_>j>_`S3i;_z7&(8NpAr}%o^-fVs?{l2i#2R9YXXilV-`DY z+eH@i`n!x>ZwF;P$^PX(`)h~%c3|j7SJ#(*((q4$GIUKFUe0Qd5kKv zUu~;W>hn13!qbGmyRx1cdH4$KZ@d^;ju~O�-VuUzeZB-|S_vKZ<<;e=`%$A)Aln zL(8hwGOTl!HP_@8r>Z-G&VU zX>32UXnV39{rq0$qSPNi*$ykeVQ|`?zc6!q%-WWN?B(}qxbbHfWW;(*%RrA_4_KMJ zZS#Nlrje8VEdME-?f;vveX^b^&<^ll74`trNPis9*0rV`NZ#yYmth~f)a)zmdCOtw zb!B}1w`{&=SF5!+hhCaCdq=J}CV^{4GO=o(ks;d2^7CU>e)YG_I_eBiriV@ls}nMK za8_^v+R+Kk^|?^ZBGlEQR8>c1yn<~R`+#-FFy3K*w&=bkDt5n8|A6P1E(;2YcyAom zsRC1Iv##xyq2>HAD*W)+f3P%YAJ1Q+PVd>MPTg`Y*3BEly4kG3;*dEB4S{8xgijcz zLD@EBkUYmElt%CO&?_~z>SXWQw+=e`go4B0Gq^1e7{7KsX!sFOu5W(^N*fw~VB$}Q zJaxar+N949Zz`T2(r!8SHGyaN0_3U3@5^WNgA#s-zryJ10F~vYe#r;R zF8h&@y$_VW+xKI`4}r4$KLw@T-?!y{v9-ZpZrUl!J%_ViIGVYVV?q>#H^{|nDB`D?n^N$40rBzqI+=`KMm58 z?a_bo5}YxdtWL(>`-J?eT>b2d?6TC-jwNx_-O05`TdPif+?(Q&7h`3i+(B`)5jL$q z348t{cn{TsIkK%5XMHs)|2X!0Phh?|)%Zp7CFh5F{weP9Lu+f)+MCwoSLasF)@7Hc zmUS$RFTpq!YTy|Q)-1?_>z0OQbF!f8qzmaDpLy4y@h=s9Mvwi=pN!wW1xjDadT0{Z z+NTWeTu|cb>niw;Fv__@SzHQje%#^;{a5R9T*%9So zQ0kHI^*SKKxJi-su8#9}Vk#(vzbBd}$ zTaHwO)TMaF*RwrUV@_I!mjtKa?AV{CJXq7N%{jO`JBn0>@OvjAmD9R_7rcD$-vn1_ zd7MA-9DO~`N$N3=@+>=wI(-?Q$Z4j1OJ2X5yVuN9htWo^vvt8bWm%`;7kb0-tdVD+ zi($timA5>zz#PRhZEcZ!BmQn+EyfO>`xVY*ZF_Y6rt#dxYHg+) z>y9q9HuOrxae~+!3}{@>cMMUrB6N;2D-x3ygHfTiwU9$!#X0_3oa3*BU)DCS$;t9h z29!S=Q2w9Qn)08%#FYQ}pe+A3P^Rd1bCX%KjoN;PXzfo@V(3rGkUHK`Vi-7QG4AQp zz4P5HY-&OL=a3)66X4kc{{l*#OJ8Z`^%Fs3l^a`2@M6sP|5KVP zhwQtvUoqp;#?`j}$~J|1(rxQ+N$uF~Z24bP;p4LPvSG1WiM8j-E$89fuSqRQnKmCq z(^HOw>UvmAS)=)VkaKh5;>94OcDV$e%H9D5iZv2ryVOg5UtCp6WfLr7d-kKOg#up$$hY{?s##d^0H1VwS(c@`ZV&3j5jZ7}vzTH$DF? ztyW97)TyPnn$f@UlnTDP()Z$+zMlOlgEb%C3&qY3sT7{**=A0Zyk1}6-IsiqOS_ML zwdh0VJJL6wW9$8xLD2K_zoclHE_gv^JQPWpiV5{pZ9j~lZ=i;6V z!wsOiT!?!03r@wR&WEQ74|UoVqb)6nx`5t8NVF%Uka&w!;bJcgc%QepiFC z+)kH~HFxz;H!0osAd9Y}ag(;(FMo-V{}`zDThKkuo9w&-S&#jz1KzXh`ZMc3gFXJw z=1iP-6KV>3$fMuzw%^Nk8hd5C%?iw2HeKc|v4;ef_5R{rR-fL#dh<>R?AskqyvJ@4 z>A!R70eT_=_o9nS?VrE5SjG>y^Z-4x0{4i+345kQ`sJ6DFV~pB{iDMPJ?$d>8J8Yl zPgLM8Ybh;PSfqd4rU#AUMn| zWHYPqjtOTCTk2KK1B-I(6El!S7u1+(^ofmo4E_)(!*?uSZv38r{DoFN3;8Q9H}ZFb zGCXGaa`JtUcXRVBj&bclPHY z|Ax4ce;AY@@-oAhlb;sjkxEpm4_f?FpbRTpjeKEzuMewwoG;fmFUv2<)nxG=Aa3@% zwQ2&`ui!b1OaAL2D|z$2sCc#%6Lrpf9@%U1lYPBGVEWrkdch7mF3WXPK3i-@b=O+F zZF`~R)oMBRr^}o5eVF)qy`HGSdIIv zFXlPv*CyceNZ0cd&r8~BRNa=as%u`FGk(NbMNhSgbcR$6&*5g{N6DM<+vKQqumKJoLtd+Iv0TkfeDHgubEtVbL_HFBTRS{>KtXwV<{ zp2D^zS;uo&=Q4j+Pq9}Sy{x*WrFP@Ee?Kx=74T^g4oeLN|SNrK0lj%E?vqWZ=Ca$qfV|IbvT)q{OtEtkiR8feh$LwIILOFpLj14_c;r9 z<_hi0+2p9-cAW>#-O{*kCqFy9&I8{$!FyJHXY?NTJsl}y`lYAt%i8PH6{J5X&e<(} z;`ir5X<8@lm*B1)-%+eiqa1bEk8a1?bnGcYxF>7hWkyY(#M-%S8L*JK6<95Q9C6wv z+*3pzysK#Bd}Wq)%FnKhj_N#p>!Px8zILE-zBs419F?||`|R3<(@_rR(*L(q>UBC~ zoctW*6DQ-S#LLgeo9-{SlXFz+bXaYdug#T>!~N5?dbBsRl?PmV^TqkwyVm7vTRP4s zr){(SwQT-DZQIv=wCwT51?da2%Z?rXvN^wce0dys`}O#Fr&pK7*SBOHo)}UmVzV6D z8dh76;N1eoSbaCYc}a&}ui`9Z`p%I0*uR9-=!?vHMe-ru&2L_m$4Nm{i&#e3d^~Y0tJKOPIBP@s#y%BNb9fGEdE8?beCUN?^}=H{#XCVAb}x{D z+&v*R^68Lz(8&otWbQceZnC)-LVxtss}y@K~i9$cdOKO0iN z!LyU{)QP($v!*2Z4c@s{-uz*4Ns5Ohjp}vXWLj96OwzsS?v7^^j~^Qg7K&r3ZE)ko}WA@v$OTV6`vDCOl%S-f#xjHokxJIVN{R>i&^Qp0#A zv|j2tjyO?g7u_3DU&3?P$)nD^aX5?fU|^TXGeljj%kxXKwPs(vrB-FX5mHa!X^Po) z;E%KSB33(Y~SV4@uQIXHl7ZfhvW-+6y-ec z0l1$oU|fBdq56AgY4g3azJE3Q<|y{x40k?k_Rao#voVWv`|IC1oH)m90OsPKd(WTD zxwzTOxO4IRH!Ibp$cy{+81j|-_6<2Bp8#ceA1FV4)W+AZ;r-C1D%@(mDZ_Ju!_en+ zoc*6h{=J|KAGW;t4j$}i_I(FWpVyc%Se}DV3Y}WdFOA;cfHEAjeEpimYS~TtE^Oi6 zByAe?$@d&F@;?G)IAVFzwhL<=&%LiWbe!{RBj-8aLHxZgUf<(bu4->J_c+)NbbG?P zz#K43euH+u)3jO1yLK+$B^6{d%arC4e{xMOzLUBJcW+x)rs_ND zqVk+Iau|0_{$TxfRyFRvVNZg)!R|LI=KrmDwK7{ge z9gp|2E4D0CD{gh`D){q{A$2bvb&2(-;QeF9EbMUo@|Gt}eg&U%9PGk*AO33g;?P_5 zy`l&%$OgSfn}VM0u#vjPZ2Or2rA9X%h5uyAaULjIr@9=Jcv)}KhhxGvz}1f}{=Y#P zs-G&AFXxkKV2|QijUV1w4_V2ZcD+=8XNvDX+L5n&|3KT{^JkOK=Rp~=mTx>0=iiT; zx2RloQ#K0O*8szEP5SD!`Dvpk4N6UK24#5I@_)4HavTXm-&M~TIq93680&9AI)hA~ z5$Rud>B~%ga_pNx`jWp$|Cx8ng}T>!MA6RK7Er;3^b@fcya~L%qh5NR+WguCjk zxT9{djnAu6r-a_7)}&6*cMkYFMTIoJQ@z)$S@7Dp7=I4_ehOsjQ*{LyeP6d{9A8qb zP%pys;BHfgk`LK+lK;C4g)s>4-=n--oTj_hIW6 zSE$W+_FRg*Blta=V=Gm|kyWbh!PP2!-&&m(#k#A5=?Vo!xpB#7bG5u3$MkpT@%?%1X;wCSze8{C9pYUJ&OdpV5J!P_Y}I39hgl1S zRjj^3y&BKJWu`tPZ{~8G-T1ywUzp?l@!D2zi@+R`*T( zX2_|ye-x=ucUXOrH*2h-y!7o>_>SRwkmb1AcoOe5Epd0S;_hy&e{QRT9Pw`cX~24i zVHc>@v#A#MVQ}wk%Nn)ffhBo-11rlLF&xK_V*W-G(zvW0$7gN5{lv<*Z7}-Y2}*sp zpHZR5_3AUN{xa_wv(Qbvv+K*iGt4IMQYh6<>#lEB8x#N!B(tEtW<<){bue1KjTR%0N zXY$<#%JRy2-q}!(@5IUPF&_Sw@r~-JRA)cW$Tyx?{G9^U^;YEL-`2B! zkKm44&vJ|v%hUw0PvD8X%-HYcc~?J--+Eb&b6GF8enYL=fcIag;(Lbmt%Z9F`nQOh ztgTV&*O;Z<7a03~4$2kF>01pS)GnOwgT522pr7HLFZ8x&dB<7E0 zPQ~+?R^xxkJ0Cb1O>aWmKV=E-?VN&pKPTgNGEP#b;5$L5;QJT+uCsZ!=6k20_fEdJ z6M2S&`?hT-;0$Rs&XDl?1t}-*dw=4_c+NjPz_zV|%u03Q<9H{d$BPx%gJ2(W&yrN* zDg5mXb0?8)ZVWyw*dH|S(m!ytf1I&pFV4r8SE#?k^Uz-7r?`hleQ?eeaY1Xma^HC< z$-A6QLH5hBQ-1cZIh+qA#>Kh(94|5Qvwv*XY4R?iUE4W5y58{~ybb$i+)*w0JxQCd zX^Xgn^2!R;geP{DZHt00)SuaT zQT#m&y*D_`jD;B0R3p}w)ho>%U9f@QYmVM8&P6bevH!dAU}zBS4&lw99M5h8WxD@f zRv0>@+`U6aE_B9X{H_w#Jed_7r}0({HUxZ!IQQ-1+OfmhzR z{@x0_Us;a5BYr3BR&)O_=99nqpuwL%YUE!9O8L+ei`5hG->-%L0{5e`-(YWH`g?J2 z+qePi$y*ES8NH8hfDQ8A6pEwIF_;gsdh|W3W^>Pq_7wUc`*&+a$9ncZo}Xh6Hx1dd#wyf2^5r^2 z@*&+HD$MUrc9}ccdCZ}*EzqupH=6v<1f@NC4EE&{M_L!2j%)2WEO?K73*EcoxU*f% z3vRrU<4G95A=Fis)9(*xKlUiOw^XQQZ^azIych9(LM_%NzIfL4F^q%vnX|=9fWH#Y z(^>|4`V9XO#I3&0l!56*9zKA)M-IM`d|_S=u)7}LyI;Fym8yB%eBXEKd#3MCqfRe> zyYcHMK^Yo1E>@>z)?oi}0=}b%IWunRW(+bnBb{}Sh1?Uj7`grLFmh9%OlxeeQ5*0b z%GDv9LFB7*@^`7)q2pGoV-`9XzVS{Y_ZLux7yg6c{p;eWwK-?y9+)url6kV+cL>i2 z{(TOP(@n^?@$_o7BHC~<{$XWLcd zdj39OJXG(nm&K)d#zquzvlO2O5Z=sT0f~y7R_A_kpq=J_gEk&$`!@y$tpL)=h&gAM?2RLq^|YpbW0R`DOC|;kE0A|7YZueb~q~ zgKE2cYm;(*I}Cl&Z-27#`#);*-2zH|<>V)Q@--hb_#38-{6B(f`I9mJ-D~>#C}dhu zM#ek;c7ELAo`*Bl&}j;9u5-1s;cRsZ->=3Rfpr{*+!8pSa{4XCV(NDSlyY--8U7Jaevfw7-Nw#?pv-44WI1nr#oFWl9SPbj&vlx> zGcpj-#E%zjB zclA!cF-&~clvjPul>gJ9%(3zFrrg(nQtzi`4DLQq`sLg&nDkeI(vDAC|D0m&_4|iq zlKvUD{;B!mb8iM?B zGHe&$v3B^&ONslYu3og@9-b}PAP8h*A1?IDaN7G%=dG% zz|m&*5&nkM>d+}lpX$t-Gf}?Z;yiT7;#5zidh%YQXXQ893h+BBpwwaHPYRSzS@}B1 zANf~<|Hfe>|1c=!8=r@}R-x0q@=^0Q16m+oi$+CU1UeXQ`j(M<@BK#4G-y%J`anGq z=y?jV#6_*1uJ0PTU9(0{0#xe}@6G11m%H?PCSLk)R_yU){FI27@AbwI|EdQ{^|vGb zP8%=JvyMJcte=~HVC*w%7;_(+`&-TnS>T#&xnd|6?RWFa!;m9?;t!2{GbsD#E>Ok? z={qF!IX!n<{LihP-#Pi!{Cft)-=6IFzBhj-zwt*VzdfMLza5le(8mA0$dcBjz_k{w!>pG7$r>kG*&$gc!xi^C{+-7;Z zZYsQgnMV9Gcq04l9#QaS?PlHs>GL4mwVeEEh58RXdn{h^ZcS$7>=~3@>-o=F^t{%E zx!AW3lfSL49yGpA!A~FgPg4%Lo(#HEG$+~w%l+|R7(e_5l;N1=Lv{FW6wW8>TJ35uO@~#HI)y#G=jl8adPS%g}d)Kc`eqRM;_=V+7z1jOz_n3Y; z9w6WM8zcW~P=+N(4exxb>mjpP#Sq_(XReR?Ud%Cg??8SCPmXbr_4883oC$mH&Pd<7 z{GN?Ed*Z*VQ18dn*KhKcykAGqm~t3(?b`5|=oh#B-jx5dpbXyuWxx2TjStc_DRfN; zU6kLLH~MyiGIWDd->YqWq3&I~#e21NI169-fcf1j*83>*zX~#}_er1r_y4!i|2?V2}wKMbnFpUnE~bWmEl1(f9q zYDeJ!JKtgP_kuFaE+C)#NohGZK4tQI1t`PymN$LRyc>6Q>c4|D1AVT{&TsGev-KM& z!}l%!wB?sQZPNAclcbBkPci{rEO!pRW9R^m!R8%r+*pft-@tY+cbdAr_CeOQ|GK1+ zkASjWMnS152FjE;DA&8Rm+N3j!K(B#_`OZg_kl9~LCgQq@|*u+(ystz`r9l&W%+Mg zK4VO4$5+S#=7V54kxIc!~*JhQ^@$Y8q~}Qr_fk)|DKS zCq?-ul13Nvbb0NrHlMEtWw-;B`FzFle+FfKs}|ex+ws597hkYMZNR;+2=0BI%)22W ztOfM>sy(wo{_cEs68W#jSvB+iHYh`+#_;9pFAjM(CQd`%+5LVicNZwb_d#j*ube(T zzS`fDwCnvt4;6J~g)Yj!w$|wWASlBZK&kJ38*k<(+(}Ir*EuPn&(-70mKZ&0P=+^J zUe7;W7)R{+rs-$qJ9Vb*7k|5)^)~MtV_(12+6T&Tx8+TJA!S(^ggcm zcj-C=mU>*f*jjJe)3u-sU7*ZU+P4+5?{@LRzB-}vek(8SJNlzab;e4o&u32-X|8O{ z_wRsXkn)Fs`BfSDlu!Ppt4iftfcY%qb$NXGM||=tRvY=k_#yV2^FORqKRvl+|yy`1=aZp`6vceGDUQJMwq-)ShH; zp9f`l%<{T?t?u`luzqR9omxG2EW!O>+yTaWNPYiT@AL3oZ}6VK+r)9UAdhi^@^0;I zZF1i&$o-(W51+MTd=8Z3mur7=%$M`a>WH!X1)#+52jzG_Zux%&CE{UFmf6|hsQ*2~ z3~X_}fmk8`>(?9oH-j?#0hBm-ZZjcdUwoRu?E@vwTKEuWni^!Hb=Zs0pO zyuMKD`S%Nsh1K#S_^lY67fqjAsUE^JbC(^YDI*agbrUiRzkQuK0l(>j_p*2w9FO8Rgz&tMdC;C?OEv!y#NA@^ zWqOfEYlXZ=4tYr4mD#l&w>O_XzfwJlr|L$Qhk6z(Zyfe90^{bOXQegvZv^4h^p1;+T*&>6y!w!; z@3QL}$m{z@x(Yd;Iop)5)z3`8B!KDJT0)$mQ`|3t0xg zth2X5TJTaLRS%}cbOD1QXlip%kLGp+n- z_`Ff@PTtvGv)255hqY;XdsTv}UxB~R1U;l$4!3(t`o>~@k7`xXpC>FrnZAg3 zyfcb_pUUN9_|cCySs7A(mlpHwGTavm)nIIDM*hy%;{3+5 z7iQaF9b-@~VEKLb48G^y6UKMm7vmfhXQ9m}09{6W3}&qJ+7CwUh-@I51jICKSuM@#kK3q&Q*S@da!zP~Hg!(0zR z|KuAMu-DlUAkVf`uB`0=_KgMD7hi7d8;1=+VJtvTw$WQomg6YoQw#XLuFl9zl_`e` z@B#H627V#`Os_P0;{o+e`TVoV2bC|tPjdmdSO9JU-!EgRD^t!1*y(SZv@?wOa&<|4 z(~p$4w`rRf(_M%L*ol{#F1bMSwmw<{>b(xAxq$rgYfANme^+YPm~FF#%`W5ym{tyemjRln3$+`Z2M%*l9JK%#3{K}my#h!w#C&R~R<8{E}Vt~q> z4PW4S40UtSTdLG8cz%GV>8(}jwRpaYr(qoLB=FpbX9`aa&&sz|sa<#m@Vpn#FYugw zbCo)X=UzO|;5p;%RqEw<-j3%xcvjs~rCRX39nY`utb0e5x)jeF@q8JNdS{ht#d8my zC-J=GA7BrjC-5{+U@XM*IXu6`bHcl-)HQhCh3DV#G~HUI_ThOio*&>j^R_B=C7$=< zc?i$W+pE+k@SOPWD)l-%U&C|Kd#cnPJfFn#UwF=WFUpMP<9L3Ir{++VisI?O^A`Nt~NgXc?l{s+$s@2FB4JpYd8gb!4y>+yUY zPvr-z)LuN}cz%HA#s369;F-qrGd$r*lmQQBYQ;Amcq4@K@x>~PztUN&@O?Rjzg??V zs5*takqY+<)d>pU0KwebsMe~J6uwQO)~Qof1UrY*)ah!2IzyeQ&cdC|vvF7WTy>t> zs5YtRVGp+%=a=WJE$aE|1?mFaYknc#RlG=Dq+YCEqF$V#dWCwWN~jLisk&6R>QPCR z>FV0tys2kkK)tlHYbc#bbPbIR_9S}}uO3N`BohaQ2KzHZ>ET#FghfK_V8r zc3XRV*S4#>hX%Vx(&^-2#vx;yo6nEMu7kGjSC90klL3O;AV}@~gZmTdgNXwpndE4! zXLAgKOd9SUN+JJ&q3-KrTb_@|#1+r3FP%B?92F(f$4OllOF4G?S>r`ypwf(B(XtH}G(?2wr z=;_RK##kFtv#+tFlc`d7!p>b@dlCnd2Zqu&2145{ba*(K*7^3L$&Cayqxi1vJ6obr zZBHiC+1*#FFmVMNRHk$I`a~xhX=kFVb2w>zM7eHAB@>+k{rd;e;^4~#Ty6~WO7tYl z%dmg67c&{?A512O`(G1Kp^3}dwJoU1{!At@IF#w{4e(BTOHt!scXA-HKRq;(N))PS z!QvY%x-0rx#^Zrh2JTJtrV_)+Oa@&tsH+#eSXRx@;BY40**};WPV^*uJ4XgGiNe^> z9Z;ASwk{x?lxw(`YL6A;6PZ5t)8RxanTE!ohHxCJM!GYl-eD&yHx3yq;FrvesU-3p z84OUcJ7KJafkFLL>)$^x)YUmaUu4pS?y0qFDP7^iHx70On8(^i8&6!bS>x?6>#DG1 zJO8Ki?f>- zaswHCU^?DksbYb_tf={ThBJ0+JDv%_awA${5a>ywr4I#kgoRtl!eY7(f~uPhrf2Bj z;QlnGxY8;%!=jvL1ExEVjzngt?0i|Ot<2<3jmbg1t|+xh`+)_r&8Ezd!x_jmFgd2X znd!LoFRTQ!cp}xA#^{c5Hr2;jqol*JZp+LqrpcO-IY_clxRP0kDLu)7GK+~?pzWqmSlMV&HpilI6cDs@gq*v$KEYB>L=$AWh+CMxr!13W=f2Oa0&#_ytxMmA+Jwpeuwo4?@L+tR3>c_;qx%rwcnAh}1E;&3jl18gS2jX@v%I%!4 zEaroqBR3EUEEp1WE|1FxM*Sb;!ySh!QaZVx+oCRz!T&SA*-Wrn zd(K%D?LjZRD`@eOW&f>W_ zjMg3K9OYbS4l@EbEGWnV^PbtX`4*7&@Fbw_IC>Vc^PZicSJ5D+@J3c#Iyqcs{<5p% z5;Qln!TIVvX(@hauv`raYd9+;0}q4Dibv0&$G88}bAZMV4)v7ZVsY3U>cU~b4MFA= z)=|YN(}L4(L#fY>ic5-pt2k(A@W@m!ZNaja>g`n<ZBt!AX_z+btV=V+oAN_v-Wx}x2}2h=JZ z$>|$U`8v|aYG~{Su-GkoxS_||z!e(SnnMS1G}WCJxkXpy+>>gS?$B=y%G0ay(@09e?p z;~eUqW$P;U^u>MQ0z!JD;ts`%FA<81TF(}j^`|8Q-*N;r(*-B$#HFmxpe1%8AG7dU zVE!(r!Sv#TeVv(DS0~fn+{Rgj7#G^w1~U)U#iq7;+j5O#C`5A+%Fu)eOnRtL&U`p7Z!2zh|5E^?juFn zwRcykdEgoZ?jpn~#N|X>7vc^hE-18Eefum;Tlxk|-(%^AE&ZIO_gVTAOCPoLDNDm2 zHFmDEG-_$g(yJ})we)qC-fZa|mfmUU=Pb=y`mm*cv~>B$On&EBy3^9@EbX=Q21~~+ zowRh?()%oZ*wV)>4YS@68Z2$Hbi1W(mflJKBg|NO*wULUeaO02zl z)6(x+y4C89So*Y$zr*U&#^AqJ^aX|sEj`WBu%-Jfebnm7S^6zYmron{zgRwP@eMX! zE60C#+H}YN+Tt94(I<=^$A8AAf7j9vTRMUM$MCN{`Acp5CQI+Ibjs4S)!$|LYRj** zeBSauwDdkpXDt1wrEjtLl%;b%J3eS}Z?g0?mbP1Zv8Aq@D=ojy(k4rrEq&VRf83YP zPpVqYE@5x-Kxe8CCzHH#vqQfDDn#uX-KJE_qP@w?t{d<^U8(x&Yp~tAcqqLWqxnGc zO8u61*QmN(?HCvuPPXWCMRmW`qlL7NA1`Vf7>1gwdHklHF3sl}eDl{Ug_sSYov<;J zytuzBnO1Yuaf#D0QGt(_5Ai(n;{Ji;71-^=)?Zcb8S3oW)8CcuOy8(;zC>G}P7kF4 zMj3A_${A}l>h+A>gZXS1kxK0&PrchRI7;fmZGd5DuYs=4ba!98Gt;+CeTSHMdZ>Gw z*$gZ7&Cta-owT{DAJX1!oF0|>5kE6gZKc!>^V+LTzen7Q_r%xl>BoY5SnUq&;#{@^ z@9g`#I|ugS9Xh8om8;y7?7Sga694AP-Gw7de_|VR+BJAXe|l)}0BR4*@&3*(6iHoB zV#Dqpr3R`8a75L8Acc}1k$wdKwMBbzu-Dz!He_tZ@3`8!@#k7qiCp*IIGjly*ge=g zq?$!ubQVqx)OQ(YnQgr|yYA22csWk|lzM?cyPQ76xGQjAbVE|;eFE)w7#lr@`f9;K z(^@{Gb_@;jG}!>!hIBNVs=Bjs_ppiFW}xanY|At3WxHCh*tO?;a{)!oEJ^^}len=z zInbjr6)z==eZ-DLl1(KxoaxawPk46l=0%CiGo$*Jhoo68Gy?&hOkOdqu?=Ha%rEV-BoTiV4UWyEPQzDtb8B)?2iZ|2S zu3|)@cc61W-mhTmrC;K2#rrn-F70r9p;I^o?O4>QJN!1Z4Q=OD58S~? zTwzUkpC7Z+zHhV_by(m-evIuI1y>cyMVsF5N4F+3tPBg%E_~Ob#LGsK*stN`d~c#R zoy5j$u=_^!PRO($& zm+!rLM||^^sJ^cbXUb+#tT$x`2 z@7cTid;7cXJ0Og8FHv|?jMER^AfrLLeHSV~;XatPg4YiE(Wh~n0#*#&m>B41sjlA9 zdes$eyD#5WD7jLF`zxq`B=x%cEar^_ZlCD{+(EMuxQnJYMYxc#^#X;vVF$W~)0v@z z-NWi1SmwcQ6yX;0sWXFHVu;fn5cTHL6Bs{L0wO&lDfKQLiwak_@zZb$qiCJlkJ)7K z2F{%cZ0K>btbeF0(K|BOjhl12l>I|6;{Xmx6F2mx@Dd@@iyLctFQN{r-rj+cVf-M} zvJ&jD#g3_79Q+L9mYq3;NAu*3E{r<+Rk0Rds&KKT&z=D#c+agtm%g`uKkvN^c4A~X zh+XG?SbDyK&)r)omxMC7Q61m#_8@R(I*KNl;Yg1g`wGVa`%z8UQP~uhR!=HuLH2F1 zOEN<1U<*8hBNiya?Lf=n@?C%T^$vPxDmQH_9Z)08wWOT+`Q5D6<@mD*hz>Ff!#F+_XQBBbgWS%z;A`{qJ;T! zLI><0lId?LKj&^6?Ae>@AKbyaIqFf)P=%G#cjf8^Ynyj-Rx|4t)AZFR z`w6obuSKb67`JC===zb=#kS_KL`b7QS1m%y)&2h;dv6}tM796@pDsXIlpu&zP=lfr z*``pSEJ_f>f~aA&B1%eIXr<6t3RXc42v`v{3TS0>x~F>()VfDSt%}~Ls6|mDpj72* zMAV8X&+BB)AjIPLd!OZx-|u<2_wD;NlTXf^ZO%+4JtsN2i&C*EIH-H|2pA}4h_b3#^UUp`SKghP~DK7K{8g2_UahigtYtX?qMpQ z>_;z@_h6F5>Jw^x%0zl}5+@e@o3Tp{4g3`MX{j(4QtS1*P{%$L{L<~MCa?dDVU>Ac<8)^1Vp z>Nr}Fic_x#RsH!(jr}fF54y^~uIkaE>esSd#VPl?jcu=rx87@P>#95s)eqcvsr{-w z?W&x1)n3BvFDhSRqY-bdG~A}r4=hsq*Bb6u$8%*H+YZ$Z+JnZnbG6~RsyDx?e}4bf zs4rD7&WDX{m+F7~?ym8=N=L6Swyi3T-*67S?z+uzZN7?EuLo5>;`dKOan54Ht*Rdd z)Op}E`-3WvL)9n0Yit}}_4k0s*ygvPL;HiOe!Q<6+ZOdcUA4FGabtf#wNp^F2fqPp z1Lc|jgemPOCvp7t#KUFh091!Q*lWu zPFHcJn{mlzdM_L4@jKI@e41_^H&4a+14f*vGvbcBDo25eYgXx*=Z)zi!pNUj#Wkq% zn(bk_8P}lVY^ogm?s6!9rkin1DlSQ-XZA1C&A6ERRsEUutlHOX=O`n4B38xKt8$p* zhv{ZqoQm_S{Ml7MFkM~~2$1JHMipx{SHOCRt%{X1fC7JnC;qTGyY zGUHTxnB#`&X8#ip80}W?Gt%QXuS5O7bTcki#aUIonCGMEW zvWj!7{F&p8>E>}=D$b+&fkl9z4&id^`h!GO1VeHH3W?ETGe%5y&PK8 zLb6ZAxqL>Pd4CWd*Qnx>)p4y^Cm)yT*r9(x6=zlDaGfG9=0T%9@>E>l6meEFPSu|| zj+^;Y!-hyyaRuga)p#EsXIF7fv;IyIm!aZfRX??=>ri-lE*0l}*{Hwc_X}pA)qnyO zr>S!I!qO8e&TAf5<r#QROwSr{QryRSt_Phj~9_#<>*4xQ+fI)Ol%MN5kW+DlQN( z@+Wdm?x$vYi7GBu)r<2Kads8g@UoGfdEXa)T&IfDRh(7bH-yKzRh(DVi+O($9_LkY zRuyO77lp_9Ra~+vhgIFThQ|d}oJZx)yuS^Pvpi(fbAgI8?~B9ZG!-Y*ac%0pFFYqhl&fReqi25hNqXO;(R_MJ@fNOcwB*sv#I(s?;FG8yeh6;f zDlTT9IoMRB=94e^&GJhB~lId9#X(QE}$y zsqp++)*AI3r{a8J>G4=O^iNZ9exIsm^*P8)&y2IHI9;WupCZnw;%usY&Cjjj>E)|9 zVYZw4ToN9qtGIks&*tZs@VI&v=T-ST{`t)eROM|nkE`-$el9n|%di@L6wpGU*vGE|(V`l(Gl-wBU%skkPeQ4aGt zPzsNw>uJzVNJ zS$Le~5u@F-FOBqqr-+MFab9&?`#mR*<7PidQgK4HoB6yb{J6;~&ZFYY=QZJRc`DAQ z>cxD16&|OnIG0M#d`=V|SFhrND$abq6CUSRaTYaRna_E`&eJ$5Z%%f_2 zQE~25#939GRh8F#o@gGot%5`q*Ra=UH+{{?$2H^ZDlVYXGoSB;r1>${mOi986KCc;sQRS-OT6BW?X~O|%n9&}VFOB;1pCZnx;#{g^qlKX zK99^eyNXLxalY`lNF(J86_>2aYd+ULeq0ror{bD@Mmfyy7mmk8800pOyU&R8s^{e4 zaUK;{ua0Xz?+%aisW`WaGoPP_$2F?Be4je5dVU@r7gTY6RnO-43E^=u8;o{~Gwb=$ zuzFVIjZ<+B6&DPPQ@4M@rs9Gsf3|{?(=+3eRb2C4qa5y2#AT>BP1TF}y@gp0Grc?& z*PxDTf2`%>=Bv1XDu;fGI9&q{GKQLxb-SdsPn`89wI!>ui}~ls@>H06=oa@ z6ovi;Ra}GFU)1yd@Hopxqunx8|FfSWPE&CX6{nvf&aUE`R5{G=8IB)U-FG_8CFt?JMGJ}f-Wqv9HUM!g*WzRwI)$MuQHw^%wJdzVNsL6_=#?L7=GR6zby9*=&gah(2J zaQyoO^*!G4^FuvH^{R2xe2%QC=S=2vLbLtM`w#Q^tNDF_Q(d>r&r|08g899j`8}lh zJllNkXI=UnFZE@po;p9`A(-5iI^@9F&NdDijk zlMYF4s!un2um$1;E+uU~GQKMl$ol{+IMjn~!p>MUUy`sY{Mms!=0F!SXZ z>6yFD@9p}k_eH7~G_|d6Cq$y!Ht&;cYFnMYB1vsq)%)#g+oraY)wWA*&s5vy_mCOp zw#tCh+*TRLGq+U+T;{gQK)$(cH4ac&&l=Ueyz1JKB_ZzQNi(3Hw2BjC&%|6#e!YtU)I}fqNUg6WLy3wJmt1o{F(JE zV-kZcH1UMBqFYFBRL_N3>FG{mQHW`3t&Zb!j>Fbg3ZG+pa&}LVerUdOXw8ZL#pGK` ze_CjTgiz|KnK`M8vQlzb?Pt;KEIH#~!92MRD}VnRT4{*Cqh|{6g7m+wE7DTB(@&}w zCNqcC#3|b+v?R!Exe6Ckn2#^!(bBtHZVz)dHZwWoVsb6MvE_FBtDaAh-4@FknrPng z@*^j%mwU?chTboi&dgmXmz_B2JuT%kRI2bm)sI?=H|K4iFoUYaa)kPQXnwis?4;RJ zdZ(T+-g5lkrvaTJE_*H$Wlqdx%W;bqEKHfh(pOa2-x{~2{AMr8m5YSrq%4@L=4G{< zZsr1(E6bGU{#<6du`uRe)zD%+{PU!Iqd{8A&p*x1SIyYs@s7_uIw4~%r9Z0YUuS?h z{lqqXWLC=j*=Z@W<|L%14W2V6ZD3-;kkrBP1BayzOc|ubYeR>m3>}z0aM)!w+uwR& z%k^a*<@l1iC-vM@mtQW`z{DfwT4m>?&?GIMpsFHTDxX{f;Z}wvb4j1fh4anHOn*zO zrS~tCGeAQN@t*Q+E!KZl=A87<0tl)Ae>)g?Zs~Y(f=*}+6(;`twIIr=k1dPdQaY?M z#FBc>?5xbx+y438s`5^e&Oi5$RBdS5;30#C4on-OrKP3})e`0m9X4cG{J_+dVX0|@ z68`)BV}YF9#au}ig8N&~XsLWmA!JrAE7C2XSNv^BL!(SBz3-pPlQ1lP=$s)5X^H8B zv_Xj}gXYXhNK8zPADl91_TXWIQ-=Qc%QHVEXWrizNji1DSP@W8i#(xBvr@A%<-qdy z9^X>AoOJ8|B%bvY*4%oCVC7oL?<)`x1(w9JLh6czz!aVTu*cqbgorn1#yd5`xO=4x|e z|BrXrEgdK9{d9R_5!b(u3N82de;%(B2c>ITdc2kxKV(S4?D%1+iRs$V_+g22hQ(`x z25PA-kJtY}dH>4qtpB33nu-2j>@J?Cm7(q9qlLRGt2rQl1iy!ZzmFQ>03LZ3t7gD9 zJP$R&e!NvIziR_q@I=%MyKo(eNPh1M_n{a#fJdLrym8oqKZIgoAO1axgM)a-bNJmL z8H4vniLi$6KsMNqe}^()8^5cw1vz2)8#`0@U7w|}{GFddq{H&Je!BC()GOohrN{^S z@y_Q{Uf6~2Mf+jR%5Ny4Ane3-)C@~MfJ7AO^x(IlQ4H+Ezd;r_h@W=>Wr3~u2&Bn) zJOw4fI(}tO$^tv^jVMp<$2(rc41CyvuR@z(H*SgJJ0iIs|5tDF5KaE^PR!M`!B+g8 zzRbsjgZRJtQAQbqU*Dgyzz)1-fDlJvA3j4PZ>`vm-+?k@?#=3)8&7_aTgKxDP=VY(L5L!x!(Kdqd~*L){4^G7kns~q`)VO}!EXF@vdHI+^3a((Nc0mVQ6V zhou*zO|Wz?s)qge1yh80UdH2@XeTWFQM3z|9zad7^nWAKE;{5#U<(}bsk9?($6aVF zY@Nori6+82{szi`Ez_B^j+`_(q|;&-=eXw|3)Ehxrw~NI)3pijsZ)*+Cja;(kG!j zIDn5y5n>(e#0Sl$K4BNW4mH9$et9bO1>5mWC@A;i^=X_lun#{kU5E}HC@+2~>H3 zw&CCB@m@HHpM4wofvxyZ6o4K0Yp6-a;EQkPT!3Bpt4kOIVL$#g>e7)iE@c%C)E)NW z=iR}00^4yP>MQr-1Mj3Ui~oFFW8H>UCVg~Tku;^EG&OpuX-K*2ez%}dW{^gj(?9b;2=Kc z5zafA34IB6<1eGwGue+HLKWnCwf2z&9JC`s-wBOcjdKi<8ZdXf9_(Z~Zk@FmC# zyYTZo-clX3;S?O4Qav}UV@ro zQOo#)w6n+?eyf){f*o6FE0hPj@pqA1#^ClRs2^C{#&sNdU^}j(1~}w!5DwtUPjW2S z`V{#@mRRzMXP{WvhyRMKuy~sKLvgV5eR3NPc|GNq`|-=rOxT5QL{8ZH41F89VHu>>Hg0{ojvs^FG4%q&j5T$4r?8RF>&sYHm@rzy{|F8>x4Mm?#J>WmT$XEdf zUn1{5+8TD^+mIa=FAK2-CBtsq`U+)+?f99mQn#>W2lr9P2Yc}b6o7+x>TBcy*74Za zIcH`38$$F!vFDI~JQ-=Q7e9m&Vd)n&a2~>RyVLP6V(qs&- z`5C`p7k=%Bw3m#*-$A+ENfZAZx#0l*Eh>P;M_eC2p-o{M{sPK6m%fC*i}GPV{t0r! zL43^~-YfUxcYMw{54-R$Q0#fce1Qk(d$1FK0%@@HuTUZ^z0H@DQO4uey_5^q@S9Ku z?8B$;Bmb}i-+^4PAGho$53n5{`4#z>@%XQ(0oK3fKJz>JA#B5+`hoKZb_R(>G3Qg~ z_~)n#EPfQCH?qq8cmT!0(vP78ICw~i)<1DPSb9Dh3rl|hrNLgj+hOtyJMc??=K2fU z@VH;O{=ynQqnY7G#^86O9kBep*!15R?_l{muiv3va1c*8LY}RpgTIC}*pGjW?6CYD z+C-EL%im+Y2c^MoywjhwDQv^{q6RpKpLt9O(Stg~{}2%(2DU^*i0e=i?8Mh2Cv1(3 z5ED>7Ed3o+082lBw!_kYMt&I|6~SjRevcKFJ_8+urKclt0qNssNAugQu!c`UR#^J8 zs4p!24Kx<^<11Q4h^ep}FGU`?A8$flSbDqG5#o7R`kBZNTk*$H01n_0Z6d^B*n*Eh zM`1g@F^1nRzmWceA8AW|V9_o@Y(OJnFP_$(JjfXQE#!e+9U{bMXkt&|`R(B$XeR8! zUqxwhKfj|r2039T{xMn#2k;9!Mu;`A9e)lLz&<>R-(dE@8om*2hjn}(+5uZSMTlFG zA9mrdq5vGkuj6;G55f+-7#)TE_y~S4H|8Sx1AaG(h2`%=%iq!M30wIc-NC4@jKL40 zBshp$&gA!lVdzj`qOvH>7>Y3(MbCm%p`L4@-9;pNx@?8er-2cdq?19=D-J8H4*! zK*r$W9Qu=t!N;H=Y{%E4qp*(eLopYVAN+-G5uyw1!}lQ#wsxm}P$F!{Z$UQLg%_eE z*o(i1>~H{&IG6f_EqFX~z&1PuWx!7SA>@R0{AH8}`|%%<3l`@^h#n{(*6~F!z7{!Q z9p8rXVITe((j{9X#CND3cK3)7Tai!h$N#*5{J<>UD*B1KMKHZ{PBVGZCL*1 z`00Zf1Lb~vMFMrzhja#0?jh6zEIkD|VJH3w%7e9`e%2K7153|E zLD+>yP9@EF`V77Z#lbGT7$wR5(W{7t+c_TvYTHjsXT-#Ux7fZh0ZdvPV?wjFn_Nv)}cn2HGV{wEUw>jKc0(30`-jVLLFfKUS9N>Pv3)C zb4M&eRvCl8jpAVbwpw&sK%Ky>k0a)xM3^;j#52eSv(Al($foUJ{>EFdPK~g`()rtc zkp@d2;iTSR<_n1XkPBvQ8SyE~hXZ(@TWCj^HDbg%Q~ztUn^&Mu{+U2}I9ZsUI1GFF_M!4899Zg<0=V^vUBK zmofMXlmYYi@8Ug_D`W5;w{fp1WAG(t17<&hG1+&(K*nz|lj*DM? zC;bez;}4-YnDq+9mq>$yc>G<|3C#Qmu?*Q@H@+Js!OYhXy*B z8SM?Trh=G)oG|kgM8tCXlH8BqhStEWe;{_D0+=-pM8e&a9cF%k*p9p~YZZu;6_f#H zJp$2bCH(|u%>nT;Ita6_fatc0G7e=wo{M5&<~@l0s0+-T6*2Z6+8Ab@zu1m6n7RC7 zd_HBA`|%ny6=qJpIPYHC8fMvo5Ye`(fs9h>hr=jKRA+K-o_ZByxi7<1zMHQL~GjCh`j%LEl%@$KpnvB7>BPZ;`yF5(4g_%b! zQc*t4Txszfa?2R}+_kiqjKS|fB{1`v#XHCgGj~XwyN&kq(}QEHLww#D3Hr zX6})=w1B#TnNKV3M~N_VU&If{CS&kxAEVx2=Dv!JC>dtnkT`}MFmqVNER+Q^PgPW* zJeWCE;wR*iF&k-9ln*o4MQlWFnE5X}zop(_=7x$%o4BurnGYojQ8mmQPcdvW?Ey27 zQ@n{9VCL|M`?qjihMA8ku6~?)fSKzgwxj6b)F+;z(|#~>Cq=(P&TE+YU*a(|5@rsh z=v~D5EBE8?qnR*scEsFb`T)#)Me%R60%neom|sGk<$kSCKa#0%W!t0O|W?fBDUPZgW(zR;F8Cd!>q{FPyDe{pQmVRLk&mUmv zS0Fzu{d%+?mR^FIVd-sZxz>z~4*A)z6_!2}X)trk#0F%8nGYw9pd^?zHN|6I@-O$} z4JZwkp1PI%!_r-71uVT06~NMyo}m6==?l>g8G|p{M%}^ECvWF`fu%1(M`a9NgF0Lq z9rCN5PKVd>XAMc!cPwP*z_{S&kfW-U(9q?5-Q35QzlaF?SrC)?*!qSJn%(wwdzYMux)~*x_ksFrY{T1>FOCO20!_uEa4Y2ec zuTo#I^kwKUEWHlJ*rG%J0vrpomZkU!#lg~xc92h4dOb>lrQi2D`Glo!MoVGo( z*ab`f5$%DkZ&E*Rao)i;{7-Zcw!F>rG1Lr8pS_bZj*brbZn!Tj{R=b_4&Z~{;n=VX zKlfewGi=32A_pveCCY%MxBeGphb_3vPuXGV?cSs8u=G!m2bO-{`;39G^x+@Se_-jG zQ4{RPQ+APO*ol9G#O2W;@A4t-1xr5%b%%X;-;WTi;S*5;Y{%VbB&_4Vp|P;|n6^cB z8IRwDl3@q_HgdqyPj95}!P1{bF4%|n-%Yx(`xEkx3Sh^l9D4Y_J!9H9(()1NbXn zk{`)7D%9H!?M&yPAc>4pi9W4C{^gJy6 z1>~3UP4s6JfW3OYHHv9$Tgslf@ zE2P72{1a3SOJDnM@&ikM4DEvT?<2$$Xg}=3&-{UN0k-2VB*sODyau+w0la;X{J<7` z1nLR9@eV)IujGE*ixMRtB9CY+?8J3shqa$5FPaIvaUaTn19;EFTz6#*z6h<5`|<8S zQ_pfge)=!eGilm@1mJ9UPOvBA|gc=tl^VU zu8hH-L`z{GesyG|SOM#}7Zt$LQ==k<4!iLk=y}+W|BiOTB07@WtVpp7mR^UNVCiq7 zqcR@<33Zr2e5*(?A9aEK_=T+_MNim=~_&raKO^Fwvi%F zGQJ2Ez|t#`7nUB;E>ielEB+Jm!=inpn2j1?Cti-4U>|-)he#2W@%UhL6n5Z8Q1n&A zpB5?Bpjg<8_dGpPSYaFfIMQHCN8W3R6eD3fo`vkN6aNt<%lJ-_;ue$zOW%g_VCiq6 zHLxE~>>MdJ$$0#ME|Fq8?8fWQr2b*)ZM#N_J+Sm%C9OZUim|YC2TF#e z=bv(hz>L0e?U!Z0< zfX}^vJYU0c@jEY$6gJ8D*}XWYU@LwDYJ?qlS{(0%ow%bn?R72b;KxuREG~%@y^#&J z;fqib?809|Q(?c{--op1e*BldobRyc7bylJFRb^c51{8|JU)B?=QeD|kD(?RuSJTB z(NWleuSC&y`X1gZo^u)2@HdbK^J95p6iSBe_;loerKh6|Sh^c6g{7CsZCLu-NQZ+s zKd>XbuoZt0)x$d8H-S2bHT>?u)En%^e?rzt+<)T5L+B5%^rw*>mi{_&zqFYLhY9!a~vUi<@;FZr@aF>VxP zgzb1SDvFu=It<4~x-}q7Mqd8h#^cf*tsJ6ohqL zjG-^dc)S61nZmK~^0z2`f z@sVO3Y{wr&9$1^eSacO_C-+a3V*~vPcH=*wopS%xk>W122bR7a?T4lNQ8O(4M-(%a z{9MCz;o3;i9k$~~kp>Gpd7eZ*;NVpH<23r7jKM3B1NP#xrjvizg};qfz0M*1`C!`t6Pe}*l19J0aEH=ra~`r~LOtm7Y`EI5EC z&!T_B4*XfwKuvHE|0tbuNyaap z!?`e>W8tIcQWn^Sf1E*C;2?fZCT$11@s&463L6}lA1O)~aGt_`e0MhIh}@5ly@fo$ zB8PJs?Sg&yR|`q=I?}|a=ki|Iv50alri`$-HBxNOGP|mofNh_mEdu`jw~&mVPZd3QI3VU1mgwd`LcJfu+BT#=_E%qN%X-?)OqI z*op5zd9YH&7*H7I5#`|%*Mz|tcgrM+MaJ_seiHvFOj#zWYRKZhK! z5C7&d$^|<%FlHhh*74_%2M*wmY@{vZe%!u^^97cE^%lxCd1f*sXK# zgxs(wWIREeU?*ONN?*UgW%kgZTWH=v%X?AN)=f zfL-`%)C7C+%YD2T_Tx9cOk1Z?Z+QDxsDIdkA9{`P5f-mUiY;%_e_$QI`5p2GJMnYg zW1LANKln710lVTcOupM87ys#U85;efi4FmcNKc=t1 z7Q6ttVI6Nk0oacpLNRm5XCwEGC=RybOHmT+!tdYBIR(4%(>@{pum$&^0PMy8K-Rg@ zAz%3^>A=!AppmeS{|ilorT5xH{lL;kpe)#p??OxAAb#;@Tu)&QpM%`6{d2BIr~r0+ zLH|M?*o(IfP(QHsORh)ACwVWQHBdmt;}QEJ#X;DD&)!cyGbkhe1{w+b@pHeTufq=f z5Sj>!uc^-iX7=Of|44pd4S(Pe`GMW|!k;K7tm9vzpp3`+ALg8r@%U&Y z=0%76M>qzS-u7qu3~a%dpaj^D|Bf86_=WtS3^;&a{44hxu!et)+_0rNQe5*J=RR!5 zze0&w0H)i1K0Smv)E}ZrFx@hBm0CfJAnh%C9pbdC~JP#o;WUq*J=k6+j&N@T#6Go!>+$PN4OuTcSP?MfMuSMJBZ zMvbs`7Hx%supf_(jS?}7*pH_nE9^ceN_6cWB@$ulxlv*wa>7o$66L{uyx)0IVg>BL zt55;##hZ}_7UxHa8<1DV;2TlBjKSYUJ77Qli#1B@l<_?{K1y25ad8{6!_wPaKz+*n zcqS@??={KIqpSK;x&{62k`C}(;r~p zB~juxq{H4m^dIDbrH|-KSzzfuqFu1`4*h6HSb9&?3~P8kipisH@!|bxUs(D&)DxC| z)&S}pmVPs`!_te81NPz*G};2T<9DK^u=G;22A2Lxe3aM@OMh-4bplJ@i5g+)d(lDI zHi$YwM`1sH5Jlg{d_8<}0_B0FXCo^t{Z^C!>w}}jZZsCwhERWKBJ9NPLsMZd{x+Hk z2l4hp>1VJFPexg=3*U`$;Q&4V zco!ajDd*1ZW6>V=OD{qDVd?E{v?<&j&q9acrT9n+7a%HZ$?XD9dCaH=Q7*{*U({D z`a{SEufwCqQ5JYCo{V-ap-le7gr0|`KZNq&b$Hfz$_`6!HGy*#j=}pPCoKIs? z6)0nAbjUYPq%GL5rG;hcr-_`@g&ufw;aqp6?)QmcAWXWxL>hxgVC^dM?L>|KGp=uhsz1 z1IW5nFdYR?#IY0p@!d09_7?VYw=SoNyG{rOW%S>6hW59mo46reYa<$?WR&qQFQJXN z!+2Y0M`Kh1K3z-^v-s~7Vlp&dTq~|&dp!Rg2g`piME}4{G5N*=Z9+SgpJi+aMphy7 zc;dfAj>}(aid>N+GQ|Qhm((&vmPjY%1!9iKCb&x|M&tKMjsGQx*%0&T#2LJI3^DVC zQ=|}o8@WkgD?M}wnL;j{C9I*S9HO#0P6~T7p#?;(5d%rXJmy%U7mCzSUYsXHF@K)u zbTOKGA5UsisplNFQ$(7|wN+d$7KL(t;+`ztmn!Cwrd7nLG^P@HIiE+Ylzv)`r}gapNF*gYYk%Ol1^5roK8}cwK{O@l={;WCFW+xvryyaSIH!Jxy@|r0YQ!3d{|LQ@48aB=gC;L;W z2idkKzCJee`gHc^9PhnEy`IiTrSQ!KV@OR`I)IX*$cDh zrY}7H;)yR`k;hweS=er3`r`B~YnJ@)rM*%Xj$g1idtQ1@FYBVr(W$BF3n|;2l&pp6 zy{rR{r!#WEsgE*pz)5);IpBEVc>Bl!X1%dtG=lM8C)fW7zFI9wEJ-S{mn4@sN-|2E zCHW=(lE#weQWl3Qi!HO3#g%DgiDkC3q%wP%vn;R7RhD1oE-NV0%RFV?vidS#Swoq< zJh|Lao>A^B&ntJ8=a&b|o6ChK#$)lsdaRx}kLF4A*gQ!dyC>P>@ML(ro_de7BCo<# zkze7iD5%gYJQd!G`U+n~LxsPhu_92>R1vIbt`LtS?l8Jw7SGP zTU}C}y)L=VQCCo>*Lmu^b@g?=x`sM`U1MFKuBk3q*IY-{xVX=fHKfPr7ClzC>T$ZJ zC+apmNw@3CxAwl%d2zM<=45xO2LC7!E|on*BkYK z-lPZhW?dA<6j};n3$2CqMZThjB7c#sIH}lPoLuZE&M0;k=M}q(^NZcZ1;u)?r`TIu zU+gPxDE1dO76*!(ii5??#iAsp#8MJlVl9a)u~fuXSS#Wxw2H(ETSZbuM#X>9hDPoD zXLS-=Wvz;<(y9`xY@s$!u5wgmgj(KJm0u+bOCQGaYTPviHP+g=TCFy*mK{FEA+Fyi zwwY7U(_MPL?$!%*UH9l-yj6!E&UZJZnztCM+ zP^cGr3cZE(g}%auLVsanVW6<7Fj&}JD2if=EJd+J)}pu~tthd`R+Lm^FG?f_xnAxm_mlli4? zdZf3sp8nWa8lX3dvKX#ZaiOb}y)3!Rp{`MS==#)9<}Yic_chc1V#}@cK3jPbeJ>+) zt#X$a(C_NYee}9Oc@upuCUnixLf0$fTIKTO)8CA--tTFozcqVAMT{B=lPc^L$!Z)d z2wlf~p=(%E==x>h+GSN&1V`v|l=XRneYTwaxn(M_-N(BB%W z{PeeARWto9w%SU6vsEY2-!iJ5^fz~P0sXDM+DCs2R5#JzVrne(H?1a-{+3+hpuf3l z@);dHHD3CgzowD?)*QNKT0^5{Qmvi-=B&-5zZKN#^fzDV8rl@Pep*7;pv2IX(@~ef zwI@Gx-SLL5oQv%i~9%>x?aYGu9f!CbuuqB-ZuPi9bp~+6XT}Q0}IqO&d0T_ zxj2SvT3qPPCz-pRJjO;jGS<`g8tHo?G#+a7J9}s(bkXbd&=}~a&jmxHpOqeG8O;J7 z3w_C75y!oQ&OMKX`y3a&ppn+s7-L+FERD3WjWI-L%&^gFKJIfg?g@PI-X+MhNi2@~ zcXR!Zlo3G%_Muzg^4I2A!~{qjpgge^hTRI?vUe&(!jN*YbIJA)k!pvlaPi^D_BA zZFE?|(OSZ%uS7W#3pU+2tm|#VleRrLV9Es1v+>W*?)KdD^yH`dztMHl(c8aY^Xof9 z7PZp)CG2(U%R41&%Ufk@%cHN;qaq_BBP~fs*KT)ATX+AE@;6uC`_a0t@))XE5go|< zlKA#o+tyLjqFZ;4yz=t+Zra&$Bc{U{SLUR%D0q(b>XZe1np_CRTQt5-ofh4uLz^in zS#z>;(&Eq2V&$H89XgH4o}ZnTZ5=b(Ix&+K;M3FMduZp&*P=SajvhVQIwn0QH*-#A zDrs7$q~|Qo*=Zb)HRAakZJnH*ook((FhpA(dBKVG5Yb9R zEsyL-qer%jTpr0R%LD7qxFEV`ZvSIlFN}P6^;5M&R#soR>%xcnT(;+qz47{_?CKQ{ z?VG&%$-R$!?!PCs=e?_s{cy(G5qCWI)an;rXmwYwjq}n!DD9BB_4)C`HvBk$LFIn^ z?p~j4IIWNUo2zcvR-ZPcy+(%z`-7hU!0&EEzFl$Y;W@Nv_zPkL0@UV7&H z)3&_6d&^@#y#D$56&)6D>t!wc=-EG92O{48;k)I%*E{b&_S~8O`t!YxLyG!eJJ6jw zZr+uHJKx%B(6y=k>-&BAQhIsxyx7KDj_iy%H2>bmEL&?ny{zy4%2@B~Z6X%U{e1p6 zsj24;xhwPD7b~y0Y{86K4;J3=_GWFB))ckP_WEfZ6W^KZnUa5`UQ0ic{l_2+soy!J} z>uepeI{MqEE8C3QblTeCZRb3eJhON4Gabsb#F74SLk2J1^z8Sqe)wCv#YgUJ)y?+S z!-K#1@b>qfyi3$pxGP_~D`JynYuBSY7sT$#p8Y4E%pVkhG8ZMa8dHvaL z@y$6OzOw4aeGeR*b@;1DPS}XzCS%Og$?b>nj{jHdUwy6BzSU3l3eKejozx32dO z%S-R_YbD#YbLo5Mo*vyby34@t9^P;qC05qTE{Suv})BRDoX1ncUq&nXlH1xTStS;r)b@J?y?RH zG};*X;`z}dwc%Q#K15GgH83ML*ExK^fYh9<{ygj9R7_2gFo1LKgj3Hv^ZFB~C74Hx zh=`8UF4lUQ+gik`Se0IO>O$v937&Y~Fn&ZWkBAUwyj1wZeYqP7#@1(!d$Ou$yACC* z=hhEj_Rzc8hGZpSDFy|`%U9N(lNPu&|i z^^K#+zh{Z>|12o$zrL`;l-}oOH=bQ~?UrAw+kSiF^4IUHbCtiaENSwJ_)i8NJad29 z#7nb=E!@?9hdfYf&jnlitr9_hJp^PAmo zsBQddK<>Zl@BHQCyEk`wWcG&ID>{F2*PbhX|KiojkAK~I+Bcim92!1*((?y?? z*|lQ-6-#C`GUUF>klRNMx#`8X)@Hmo;mc76+wXpH(9r2&Ri;h-?~ZDHwBEFHbX4aH z{%KTW$elcBNN6k_qQ%E2{xy~km7C^RI^%?+PMZ?nS?eUzj_J^D3_k+IV7?$T<%HpU z{DNGb#f{0%aq^or^5LBMT2zNle|wFyr0B##hQ`x~2N_QzR!1(6SSH>Yb>>Br7k-rd zNA+FTM-7O*KXUOSCEtB+9ry9Hs1d(UyzTqCroQVxuV1lj#)rG>uG?Oy&%AHy@T2Ws ze_A~J!v5+fHs;3^4|yW){(?sLv`=eiojJYamA%h*@BT`0%_Tow`gn(bzm~B3_tA4^ zUvkrrmu_8cJ9_!EDFdTWXV)N%~ zuCR|rm+La}5E$j4tRIkX=`gfm>zVMAU1@l{P zd38*;+Yc?AImT3-3yOC^72dT8?M&0$rCM~Z#_I> z`m&D;5C8O=bNlS6D{_4io7#-Cw%s}Vtp4xxxPQ_W6%Q}Xns<4(DIctTI&#RNS(_i; zx~6fhIiB51i>y3pJnMML+tz#H>JMfo)%KZM*1Phol6N|sFrGE9ADmP@e?{(=*U}n3 z&#NyO?cMy{$>UiYm)1cJMxpU6k}oz|9Ly%j5Nq@p?r{dU?hv|JdLlj|G9oHcyGXmx z9Csokx}GJ+orMf!JiC=w_|O-^};HJf&sK2jgPD)EG|@5HyMlg79c zkT(MAK48pf&X>>#c$;}-M!@;nJZ-GGtwo-YYHt7kB-Px%%ELGsDi6g8jiVb@ z45Q1$(D#F%zL)d)zi*$>ULXAK>$hJJDA?ZNyB!0|Z?evuo^s3ln?8vg_W6qYo*VXF z)RwLLCSLl({&oXrcX{ixt9}_6`|{F-_H}P-w=dQx0F! zeRu9vZ5Dm)IM{F7dOffAmPz$zPkgY@cg^z=ou62~_}UTCc|YBA&l~n1&)B=7y#K*2 zf7EyW{ND3!xoG6Dx$|$c00b znX>Ac=P&dnwd#7+vB4K?8}V5E_S`Y{vzAsm`fS_v*^929#=o@bxhG3H{l3$0UHs{& zX$NYr+Ta_Uabx+`~LOwUB5I;nG)OZ z?gv?S9!+ku{;b>52et2Xo)|YSYfPT4A=3NsZo6mtHL=GUjy&;x&&O8Yc6L+qkf_vY zr$@|o?Tg)c?gN*cb@Lr5>kC&ty|Uz&!-v=2+_mQ`sRQ17>FDQ+Ket@GH$HFj@Q*&( z_`|PFH(q*Izxm$tFI;kX+WWJ6{+M&@$>8d5Uj5#2%aD&Z4XUo`@!>`HT-*86%f9Y; z#qN$DJmP%p?6sSsp82ElKY9gF?tZWGqx9<@zw@U)f5zpH-M@TiWz=Ji zh|{`%_(zia>m${fU(Tp#v+&G6&%5=ln+9hr8di4Q=KFSE-fHU~7wzqtFgfzxN$)&# zcIB7DZ+W`S4-dR{-mQI)R_}S-w&|se?}laHcUk7U4d>THG(Ym()l)9BRQ)_-|LZTm z@^#_Wua$f`aY5?7Lz|idukQZv*qx5CF_-UoX>gmnKJB63@ZIBgUU&8%4_#V3WyHLP z=T`ht^y%+w#%GmX`PMImV&dYNg`eDc*zzxLO&zje zN#U8+jh&B0|5>nTY|5+C*WdEqL!(v}erL07ZTKjV^T*1!(uR%v?f$FuPwuJR{FEqe z{aSlRyFLdx+>#T~aoml=O4n|?FekcOhrYLOxbskt>b_;`OJ4YCe{bFL#3dW9^&g0L zTsigqbFyyMNB3A;)5^1N;pun2dT2}6pBk5Ly=(nPy*EsH@rBRgD`#x};EUPq&OT&) zyK(=mTi5*Cwc*olrPm)G-g)NrPu@Lt*00`kZoeb_%YS+IXfIwcvDL%PuU(h-{=SbU zFT8j1ilS#*Z~Ne)`w|o8kLfn7;+ho)URnLvkHL%YOaE%ll1(f6pLXfRGe$`}O%Hhj4j(BpwXR*6V@Bib0@x9-xd*YFIYlgl%>!KeD4jt*ddE2g! z#((qn0?lquyW^Xe=Y2b=3%_P_uKmdYb+Nlrw!ZP;=VKon^6VEw=XKgSz3<>7uVrQ( zbsU)b>6S|duRH(N4_=x+^7_V%@4ko?2}>sKSvqUyFRi{iGNI4Q(O)(m`L<~FQ}geh z_`-xPaifMU&$;vIt1hm&`O_ai-qP5oG2Qvw`ofxw4%gf@;KR7x-e=~y?%CZvetEZf z+VXC5c?6_o{GYjpkF>P((cy$gO*~sV`}pIVwo&mNPI$}|t#vaRCO#}ACw?r17$iYfUuy}M`PV%PF5)t(oJymRqc4HN!IZtEX=C@bs1v-NuC;8v4u zukBm8d(GxWCtm)z!H-@Z;qs(UZvWFSXQymjb!*noUp?^k>_;EnKj#ytZPohOpOw$P z?vW0Utr6!ha9q9rn|(_X)(k%QWai#k`vcl6O@4dNb z_0ONbd&a-!?|EiUQoBym#vT1-g(=5Bcy+RwlDDGyA{yZ||8< zs+1=hy<*A0FW^r$oM_kxu<^s8lzaIGKKJaIu($q{Z&jqNZ`Z6;k13@l>MDbY!+TT_ z65K}E-*R_+t$ih_+~7&Vxdl;UoG^?HBdX3^+mAPIN^OCO*n;|3%#%-x18xsp})z;k2<0WA+ zUd%xdKDP)R3EB%Ku ztOLkrY6a+?8DEZNq!fjS9ym1OBH{q$lQ^FjQ0NEoJrbP)7?J@(2WSaC5e-*~R9mgr;ll5f7j1`Fy-6#vzuRD2k(I3!K(;tWV$Dxz8TYehYV2!qirxx)Y(PL?OFq6o`<(KWoW}8KR_f5_dJZhQH(p>d$rn7L zH|n}~C^~P*D^?bvm+4=Mv0zzcL+G!Nhcfi$Uw?pDrgl+0HWW8};T3x+d|{B!?15)eU867BQa1z4OI5W~D4A_Fk^0%~XFPpb6KSC@AvfycHeg z-zH`WVq4RB`?x`j#S3ZLPj^vnzb-l3$`RdF5I(49TYAQFq*N~3LY;cE2JwplBB<9< zuLG-m|C-I z@IeEUJ`3}fUrnuh+Uj8}$3bOMwAWJgA=V_C+O>n{o>uNn!~2<<^+xCN%z2;%FvntV zx&0NH9NlN^!=fxOJj}DRj1hj@Jtni!5>FWC^y-QtV$jwa_dj*7@Oe)gwvqVGr$A|} z8KrGET{gVtp2IU=fm_jr(E}vi05q8Qh zKTBjN?vo&B@Z66~fuQYoOyL0gvvMC0)1!=(2=a$mf#~1G3ZSojv424nih1&RvT*6Y z&FiB`J?CV>jgCrR>sB9pKCYttz%xSN<@+X$KFt0*5S*UDQzs8tOeOqq0 zxIOV$Ugg~-AysE+_Ckg4nSL&xny4kO3G78CZe~jB_$v!~LpW~(vZ&^U=9dQ<;jqZ9 zuJJL`%`BFqo1Ecs&+8c(T*?|^2_f}0f*=DAO%S7s}`Z@($4xW>v9pm#v54J~d|str2@?wXaKC=%e*x^Dk^%6swnsFO0s9fp_t%zU3dp0w#N zYO+9As_nDQHPhDC;*Z-H?CJZGDKz_9@wP9W-`1V9DPHx?&VR z;nIAj&v%aiogN~Qr{z9k$$x6-|3kDu^d%nwFk%u2fFQmlymSxjK0Ce|1OUf*fK`Ue z{<}8$F!DdhcU?WS!hluwhIm3W4xa(5{9DI=(<&c!{Qp4;)MFvwG751EvK$yIx5X!2 zp-X<{-S#{j?@diyy7JZG0T-NYqgASubT6^#IUqi0@h zj3T3h*>s6Uq1)31r}7wjBA5*KwM_1#eb%-+Z0?Cgaf?gyozxasUoz!RPvSNR&#+@+ z?U(QW#Nlv-Yj>-7qiZl9C4VvWW7lqrRX(h!q_cVo|7Q70hwSXsn*uAnDoa%&gqzHs z05>tyEvuc*>=T;}O8%jqS#rlh_4_`btqV<`jR|%YTQzsiy{Sc4!}?r|4vc#JLi=P& zOm{0!?&=23$$h-5d)fowbBV?Jbv#kJL>=Q(j{}6E)RlCm1^ilOGm4VIi6etiF|}y5 zd&`*8i6j)Og_9BPBxyKEgeuQQf?Y{z-R^9|+z3Q2b>tQAnP2KbgU?L4yi`yl3 zlii5a4bzqrYgND7aJMBgb8tPNE2I|YdV`QigjAfzGv+5lDoitCe7;!nam(>@M!Ua6vZBQ zq|!21#phl%&36#|Pxf^@)~MO43eJqJbaa1TmLcj%-@}`JtDXY}&oQZxPSMygobWEt zd(rH@ptLowH)b|m*^-czv%M83C=hj&Lc073jm|o%@AvF=``_;0x&z%lH*`0n9l_uc zOT%d)Zg-=MB)y^9hCk+Vu{x@Rwj{H4*k2dN_+t8u1%Q#)0UXI#;?pBq_EV|*y_Wr*%8j22 znZZ{f(}HL~)bdpFl;Mg;gv=lKhxRXLQ@$iSzi&Wb|NU0E0c^kTyYCAg7e5p)K+O0c zyb$igXCUc#;#=oml8!(0{yUQbR6POLv_5uHUUyclU(A0peEh~joh*0UTIggCL(=;A zrb;I=O8RM-8xH`@_IaovPuI(DJbN?1 zQeyDJ|07%$Pl(aIcb=1{w{#m!pR;Wt{haK9Y2$FdJdc4Rp$EgUrAF?%n{!I4WzW(Z zBeO;XO~V=%Bu2R8w?*nNxG4|os-K?W^bZ5UJ2%`F--30OkeO`LHZkR*9*T?~%_o<= ziJvOkHLGMZn3`F6QHWq=&4t^mixHgaZ1!W;#uq9_hDqI(AC}t+*G+GPyW>SW0~d%U z^cXjRZu6KVm))jRGs*Vex8KrhhUE;Nks)TXvRn$2k}9q#&uJD-Ft%^3=9|w5KQ9(4 z-}(#3R6-7Lbc!pCxo_H$b0>83Zu6$&oO;&B)U(>oyW6i{5JHQVy@u^xHYbeH`sVF{aw<1#g-Fz+@q_)ZnMAWpVeOtF7jN~wM4Rs-YIOd5DQqcgnEy(XGk zIXT>9Xg;F&%7g!9S1mE+`V0n{w15-MUe=61fW9k2b9nCA0k|!pQE)yo(NP?=0)J{LYTf& zO8{<@{{SWd0XwW)Bt9lW2~-LmWr+Vmw|$t_#rU1iYXSpt4JPNz z^01!{Oc`GMs1gFZflu4@&}88XeDkWjDYa&tONVywE(!zjf?O!Hg#1$~2%`Fz5E9mm zPa7rgAe0;t0lhkuDBhVgjc7kR*$sgeKw$BOuvnl<^?${C08#y!r}HJZ09=v<-X@*h zl-723-|azbR-V79noAN-Xj5M;U$OTrh@4q7Xs{q~&_TDIp*|)4rVytqJA#F1xypl* z=@|jLgH!#uY(oQ*z*lr?=M|-eg#?#bv!Sg7X6prOZjl|5tjw=}GIo;Dd#JPu)85$> zIPjJ!QFuVcBpfGXijsD>B@yGE_}Vu5EkEK>JM+>kDf>e=u4O4hnwiP*ieS9LTUT3E z+$ZzzjnDWfU#`=5g6OgHs0RYoi~=QElN;PX+`$V zb!qgN@KdZeLMnmJ+AY$VM?zmlc{RANRG?`~#4D4nH)H-bqH3+SsbAt$yg4L#nRBf{ zAGD%AY1Qn!5Ui*uDU;%%WYhVGzSQi)@J;5)_($QMg!fZcmIHk|kvLxZ`&bKGe6{1_ zJf}0_ci-jFMSx1@D*7IRmF5^9F#E)UQTBy*2oRh-tMhL;oYr2>FKC`K&#TfTrf!R= zkTe;W4ovP=_Um=s$u<^8Gp%2^YZbjSO|Txxw;-AFRKRg*jH0~m{2TrRZ=wy0?Fhb( x{Fw^!W_5#L!J4GpGL&FN%VGx|1#{E%coR?er5)SE7M#q<24$gZq~FTT{|{yD!~Xv@v&e7|??z30YFZF|1| z_dNeRnVEfAd#!i9>%FgctzCchYLD0B@eJjYOnN-)_?5qa{yxE1p2st0!iF)P$3{GJ z*1Ej#GiS|Pc*|1%?MrUEdC84m@GrRW)?06j_&@tO|B~pf{#$PK*Iqu?|ApIb`rP@2 zg(FLysvmx{@gpxj@X3(-`OSg<4sGZBfmbh5zt_K&>RJ6eS^ZZ3UZ}5sOVu9z`$_eJ{@oao-xnYF5B0f=f0!;FM^{gRWpPzcj zdDeL_{UgsvwR`_N&+|8)d*MfUp2ze>PNecdujg-P^81%OPy2cN`iFY1<@)M7^E~sU z+%f(Jx$n_{It2h_zTK6hK5>5Jb1Nd;`OZuSWa&E~Wd8X*p84l5x#`BpjULa2x!jTN z2@qj^GyVdQ&-r#SPv7}8+QI#gQsBEsl;|_1;avECNKLKEOB37)`8kLGPt32`0!lgj)B0Kt|5OP7JXL%I z{}gNfY3uNPU8I1`{E>uaBKu;~>TmH>=I4JnmycjEw|^fPURU0DAITG8GdWY(P`~E4 zvo*ivKQfR^2F-_>7oY6$guMO9mo@0ZK3ayYzK-ih`=m#k@fUiqAJLapZ%2~9)Ro-( zx#zj%Pc)t%7$2_uvyL`?t*3mL@oVI%%+D7%myaOe9}fQs#xqOU(f&U|{$0($=xk=bKQA&)7#@9rqkG^Sm7f5+XTi zLtRF5*iYA%7F?xbultpG=yA>5z6}?6mQIMhQ*_Tbv%75%@HlmN_at6U7T?M9iRYDh zZE1yyJ>|b46S^&1Y<=*xloZO<2f96FL`sXCMh|0*DRW;>upsY!HUCa+x_PrlnUT_} zOfaPClHirPL&n|QGo^ug-2iCXSJ&DRxm19t(5$INBfKdWzt4{+y00**gj0z@B{>&;Pp0Mn(rv^M8qWN)#ga|!0eA@S>zvl z{0>jg8o#!U(R>vTCc6hdAvG&wPEqEFd4MhXb28@>9FOJHVK*xVS~d+GX1!EbqAQ1>UabFcU{oz}3y`Vm82*3fMT ze6{+aAh%;LUmB@iAK+2_T=}Ni=J-yvb8j%&p^|-SVprr0W%gOmP4akR2a`AwoqLDX zmUjf>fusf-?OL#OY|!lBfsi@6Q?+f3Bof=!2_Ef!trP#Ya!1M@6>5)%3RKL>i;6v} zsXqh4T>tDu=it9AbG%i{A-sFfl40wTFmtz@`aPbdobuPL;w_%NfYxahIuEJ%(6Q== zNw60kty&KAsAq|YD%Xwx-g~q3*?=dPi)f34FmJ)G>h*9}n8&Sp9f9owE z5%WkM(6q5GeMy0j zA>=wFI=r%D;5DENn?ovhgv}8llW{hFKq+-#pTam9rph;ky`5pI^sD&1Qa`twxgAVC zD4iU5h=*%^YYM;jYp6% zp4kCqj#aV4dCM-ZtK2C4#Xk_|bwp$of>Zjt?7~DsmpCtM){eC6o*-DERCr`TWQ2p4 z#DH#=07aL?JKP|IgJ0Flvw~>kd70`B`Z?4{hu%aV2g{T(ub_at{!Hv`*#c}UPfKp9 z?C?!!mS;3QY5}ccujNGtvrCJ7ySB7Q8H-CxL^J!Z3SJr1ia%_AA}s8;ueY`orWO+> zSftE_D34Dmb4HPhb#Pb`u0G&<;QMmBI^lcZ-{b&`3f6|Za$}T@}8RclNAY6SA36~gxR)b{#VKIDzZPEz+`-3+GuXhA&n=+=(fGTs2U5QjsQ;?xP zZN2Q+TZ?57u#Bz>&I?}Qj)8B@-K8t)&E38=k<#1i&As6Thr{MTy?J1EeCRzYe)`j( z)uX(-D?3cfji)GcdO#F#>!}PHhR~u0+j{an)LPn1aTyN8H^VtMpy{`5hp*M*hDgwj znj`}b508wDN%62TyCLyk`tI5F@zF&gxTZcn2Q>-77Y>GeKOc!u7H)qV>`{0i0^;k5 z?Y$jnP5Rn?LIZ);{l3=!cm%eRGgz7#sJJ=FQ; zu))jE3jfu^C_=HX5ug=0`m`-Np@T-DgZVBU1g{HTdxJ21sJy$rx_ik8u`^-^Ke;?c z$LfVLK3y`JZ35 z(oBx{!l)Z9e`}594T*2+0uw`K?O1!%iZOZcKL+c4KaY$ph>X)^_l1p-QheZZdiXX* z-U^#FB`SV-z?v?7HJ135`BvDNg^uY@#H1*4FeqI0tTLa`7P#;hW__eIAjY@Sh`oLw zGS)V$g@dZQqQl`ZBu<{1I;S97zzOII@;j?YOfC+NXQ%v8Vf7J=1Za8fSz^=n}6l7gnJO+&@aeu(EPpfKFv6E!YG3CkBU=us&=aI z&YGo6hHv$JviQs@P`{)3?$Y_52*fl~DFabxELU2lK?y^ZaXV1!F^!(foj{^gpBgjh z&~i}1uTu^gdXMr%d1SeY{qfS?nGMpQZ74qz+WD$Yx+5<6i=dS!7mM7Ah}^n6Ew_T# zW=nL9pYQ_wzuiHjp#n*9O*lTjQ_PWChN|9BC4)iVhR>(O0_H_)N3=F%)~JvXD5^L3 zH3JL-oe3KOY=#;k&}cn(0*RS+9Ysf}BT|5`0Hg(QPfdWjO5_RZvO7#&MfFo>spwE? z0>i$}nANaOdj(w^WXODL`qa*sL!F6XgR@4A-=nL-&=sc2_`{V7Z14>6co4iIzrqIp zfdUn$8BuV5)&_k>d>IZeITgx)NugdHv7zL8dbRo@D$> zu5gq%hV8u4g>{w5&;l!D?nCN@Q8Fsz`&lQq!bm;y`LOuh=zuughDv;GtsE$GHU`Gs zP@Q+Pif_>3y5&`o*GR76s(O3mg0OLM*u<{BvjD4C9q809%9v2gLG7aC2Y;TI@YYS< z7&5n7-||so^*fP?VdK)U*&8aIt`H*e*c@hg%| zY5Gv!tzlzo*ti3!DE5&u0n9e4u?w!hE|>nAr``3axCfZ(ILNkVf>RK{z2c`RZ5TT= zEB1J}x@*~ZWsb0(M5>0XJC+VVhI8$VUxxm0f-tXPh*1HKmU$>-?y^3*L^OGHf{I^O zq0HY}pWL5Ja_);&dm^Q6$%wxf7#*isxIx8p>f|xvmNo(a_HQcr+m5{_Z6<2xUKwPdO=cAImcVkTT z!%E6}l!&}G$D~6j?ZqjjEh_tZhHYk5xO#8ImtkYgR0%A!jY9PeA@eyvQ`Ijn{S?Ii zSZOn$1!Md3>dYSNQiu?2f(TUGj&-n4pHjdRbGemDQRM z8vfY3LwxtXhK=xZ^kQx4iuMA*36o`@>*;}M;Km^iege)22kFrv+VQnMpv%oc%yX@} z-*?|iE?`VA87Ek5 z8gIvEM!WHuWMATpqsL};26ZH4orri!Bx`-O9mH?6n<@Zk{g!KAx%xj)ZywTgd&rsx zmd#z$q?#|yRpw)*tHGjg%}=z=mKuNX1!+{(o7)2Ii%ZMu&8N(rwe9#pGGI6U1-gg0 zPma%J9Jg6{=_>E;6SMzX5ul4p7m8NCU8L3uzzm2tk=Dw3d;_?1rG5w^Dd0B<_>Iyq zkHgk>C?m2+#qYRXgl<_+ZD~c`6QYOA7j^veIMLqwFF@VErZX18Ekh!Hwd&S|Lcf(A z8zfYr5WiCkVI1(_QLAPw>>bVlmjw)9hw5zOH{dux5-2G#=9L!By}}U)wk~N@3*HL) z)(oIK21DjSiOcjpO^1@I~=4LB?m=>xBmulw<6j>JVt$BL#j`bbXnfN46%7Hv7Uw)5u zea4gGRHdK1WOB!&dF!R2unUWKvmjf*X`fe0PJjlfoqDWOGOGCxsu53js@((M17`}- z&0h6zY;T^b?6TdZJYVZDeLW^US9rT^`@#035bkEHiWsLdNBPzqwEhSF*B;q#xGTQk z78w&D5+q9RE9~G~JX{(u?G|AQta~8!OI=$IN$Y}}$Xsz5e63%mhUQ=C3Otc=;*V2^ z8HUaKOCQk)*j!6JKX7U<)7@)4pzF|LRNUg}Rk$pjiz+0#6?>{tGmw_)*Im<{cteYw z)R)8m?)(J8ohV_LNoaFlFg_>GxOvz=j-5O=8_SFP)5 zH$6!K?-1k*csv7d{Y~U_`utJ)kdi-c`!tI_CVmDKR#Ad}(eOzXwq;9xu>jmrQk4ABf?O`q(tk&4f3`h*?u1k?x7jcTT~q=`#BxBHM+o2F^m6 za@+x&1H~L`(?#P;>lp1^qGZf&9m}Y&@<{QL!yBqHbr{%4X#*h0!8hKYI}hdfk0+Ro zdi`9HPmGdrSsB_2(`H76^!y_@N|>0-3k+W_0_1A?8uw!vvsiyGJlAS?)YRqL?F1GB zm`toF+wGhIOvKhqfORfVBIw{m$i$JKDUj0m75yy+(})zk$;b%3SuYzWmj zxgs;_6gHP2jP67kTFM-tOLdj61ht12HpbN(^G+@Z8#uG9?1gyf1)=Iq%Rb`x z%kZ$oA3j|mRNEJ-?ppqaVDbjaM9xv=yThip9&?!D;%A7Xt|Ox-Mb0JGB*ouU#w^^q z`V#AluP2kPng!pkSh$Y3OxLpAVeh`$_PG~^<9C(S8y5janeuKD_1UP*siEq_%TCep z4J4(&GgiPk=+rMZ8b)%ViY13c&sUiL=8aYL#^=Csx2`MYWHU@zk)i9lsTp6woaY0^ zDZWtkj%5Tpb=>dDf(UfdZ+F?4T3fZ?LTpj7Cl-LEXRI3GdHTgA>l;W4Oz^ME^9<{D zxqr6YH^+p%2Nba!-+hZ1UyXJ_DXQ;eG5Puko-BWJ;Bt}m)th|x&5(oGp&`+sVY8`9 z8Dj>fXtGERa@c&Ku6?$@u6=sRmBB0UMjGoTzX8Nzc}VDQGjcFwTvHG-7Fi+VuD$if zf?f5-(*Anmu0HKwoAdqQI7Y0P;pW!5$w_NtO2qnFiN%67v3x-2<;o0{si`wdqIJp~ zR$HZKYocIco%pQL^EeQnHCkF*b=QRwFsJKVt#jXzeq)<&wJxCM;4kwA{sdtJ*CIpj zD$_QCHZOjy&5P6Jf*5?w0i>Sk7t*7x*4MB+Dm!|oS4b<$n9d~3 zbV4}OivsP7rZ_+>nSuqOjJwLTL9kAwu&=f=Y<8m$n?3vm35&%?S41MU?Zg4COG&7M zqK)ZDnQCZtLIDLOhzJ`z6**m*SNe5?Q=~FX(Az0(%gJSB^=+^C+HONuguym81EMHw z-sHyEu;dyy;^V~9^sPaQ_J^24S}lapwueBw zi0o^${)ye`_e4)s@mUvy<277=>0sDfQxt01E5U@sC~n-$#SWRGtL|JjK3v`@L4jrl z3o%0wki-dhY9tL0d{$Eume|Gi$~_^YBxE!~z@Ayg>3pvB6wK-!DuzENwVr|vg2WnQ zo~Pn6?p29GU8$yNn|1gGO;aav>Ze7ZS#3|hzE)^<=}-+b*LKjkypI`cLdwph_5EFd z=xM)};O^9G{LxcF@h{8h2YUij@~lR&17c5AT+_|v4$1}F8>d(`sjGI4V)KM5xbF8e3%f6 zghm0wHFk0psY8-SY2Ru0{n;CLr(3V?kuWnsYtisyTzw8y5@^1xKrWCH0*D#L+7VDW z@EjS>uTb%O_kxzNxr3Z3v5RDO8E)9eba8*sw8D5%e*7fs)R%-|4hjw?g^82+3B{K! zMP-d@Ua#k~qhso2UKjbg*}CLqSLp@BKT}x|-wLu;!e)=Lkb0jB8>-0qj#e6j)EynG zOmCp-5veiy1QjM8VSc2*Do17j)mEO& z79=6?LIFG8p0Y1&>(W||Evn)V2*q2AIMy-OaQuO0@WCXKF`_;da#iLHMaYE6IgWjR zS)%38D@c8rT{5>JL#&VLZk#65gTG7LMj}82|3!AX5CJ!HcVJB_>=rF?tX=C>5lOZi zXYG;W*j+^)%nhsoM8(`95KA?}>k+x-BnD;B zqlCsZ3dHR^WDTGh>{*5&#s$)OmH1qN$@?Y#Vs?|oVcy>>s1OPpV_i%r4wK#2_A@TU z-YHn}f}}(wWA7w=t)l288W7NNS<$!XLN@b zTZ^+w$t6lHw@W3v&7GaE4U7HZ(nw9%Y<;eoyE<4?zBxptJNG8LnB59Z6jzBT&4Y+c z@hwj7e0f-TfBfoX;-bz!3@hK2+^p@5*x`buXRS87cqpTPBdx3L$D$i4uw-wlZ*7z5 zq!W0hgCgyo)V?XWZg6oOyn(=-gK7JxD=YhI%a+{p6Xxq25ubV=`9t5tijkgwi2NZp zze#~;C&$SSQtsR|9K!Sb;6QK)+RvX7v- ztv|eq+^OzgS|~HHXzy{%zam~T`zjTvmY4K2iklF6guX+_9mY7fz4huL(K4BnEY@?9 zIv*mP>p3i0{FT!rnlVgO50C6ht5%U>ZazaUek3))<}~}eJv$v|OYZr7C%ErhlRZWE zLC;yL<_?VpD7Ss;BTVHar%~cW_w970bd{3Ne7yBf@8|yU)+ga2NL)DCWCTg&>C9LO zW#iYkHaN6tyOqMZ7}@cA4>IqGiY<|3lTZ;KKcBK%BUDjq1gF%thb}RKRk{)?rjo5!RR@9gTyj}r4GO{GFlI-gE^ZB+z;^Fi=#s}?`NHbSB}UNYUNi9{JCm3x%Y zSj4Bqx2CqVs(mK;M%d;QUp3P&-b$pj!kGDSM}IRthQGzi#?HNCP>0{LAyeK#9E^a~ z1ApsOfNhqnZ$$48x6b7tA(Pw%GPyAH?kB{gsjw4q0?L>+`J2SRQWUEs&B%~+fH_4y zHJ4-)7kv(!L4PI=Yf3^%kT|PIP&o@U&H|0IKw}MPoOOv&Q#H$&*I5z9p3!3va5J2; zWAY0{3*Rn%C7D8E@_t2%>9oW5cszFxpWHEUJ#s4?zvJWKRDAwPu6WGf%S8CG)3KQk zPMl)T2hYIX=!h`$H!w=qRa-ir!O6q(8)JQ6Tm$V=`C8wgT4lbi;>1irCeu}C2x?s# zuojHwYD)04f5+o%hytgfxkrJLv{4QC1v%v6d$eiI{e~2Eb z^1kRe{5d@VywiG^)+cu)z7X6nl(q@f)cR_++c^#&Au02#RIW^g@5`L9nIDr=h^4M= z$I2?1Z`vtSyr09$3}IB8!HQ`InPQ0IU0h#~D3E*Sa8I>#3&g?XCZ4-z(<6Dk`ENZ0 z(EJ4*n;W8BC?@aiLLaG_6jzQpX<4!wR6JN}S)Wkt(jY4S_V5oG3@Yf1v;D}*KN zY;y8t1YLJY%g35i0bNxm3%X2#gTUnU2WRSXMsOojYsTfYD^xnOs3&-dr#HXM>52Av z#51?aS^LBI-j(yQNd_r{&_&2H@!{X{Y~`w zrn=T-WHiy1-lkDKP29XmW_ss`jA4n~=ByVA6}X{Hh&!$S z#anS#MwrE?1A;ALI?0HD3K+V5q3z6uN4hFPv7Um<;#2n9^nB}a@wwY55 zbwfr~gg^@~*XPe#6Y!jjGu~Y-4Z&HOFLbttMvC?Xum`g!5_>S~9Q*i`mE%(Dd*Q_V zebgs(&*7isFoV0qe=OdfdGSQ{wb%p4)AuY-&N89?-g8OjO>6HhN38G8LB|R2tKR(o ze!v#2DxUvu9OmGI&~N5?O<8W8i7*Kb=iEPje-VkJwb9(xseKxEVS-bsh)lUp_~YpJ zlan?UWmq+~{g1jP{1+-B^SMOyY#PMeeDv7VWXkrJCYgU6TAD#Z_**K|Ldwl3%RN`C z<}?o6{JYdNiP-zi(?0hVP#T@VO8lQ7o|45)V^P9T%Gtg(1s4}C<vUnIIA@~KU;8)~JbhM4%cgHaeKctd~{IretdqL?YzVi|U`lhYtsCnA+ ze0!oJny2lM1G2K2m>AKM|Fm7KC;wM+S@*v;|7&v0UxsG3NWs?l$YYZ~0PVk>t>R@ay{4bLm^Jq;7p$-};rD$EN+o1R^^b zkF)pWjKu2R{6icGJqe`ObUSc;H&w@*3}Lc(MY{U7)UAZRbzS<_=ToGZ9yMr$yMDhRFWQlneTM6ik8m=viJk52PxRQN)nIc&+7k(zWmr*Oo6__HG+M< zHM^`IVsGk615p7&gGZ+?cN!J%C3=e6QlYfe8*`f==%;JHPil`nE~{ERq&cuuD)zWu z!xS0990Ot1U(#ma>|P#~bvyExW`WG+x3oVfQ&j0o+0yyQcvJ>H;)UBFg=BF^J3+e? z^jo9i#cMQu^d*Z&K zM5)+D1PJqk?s_dbPOY{hpG?o!fU{(A3yy~2m4rrj@EAY)zIkf0__N%u?v}pbl^3oiUEIyr^ z!NqP_5pragQF3Z*%kF);py-+f&r???1^pZ?yfzYp>qHd=}hTt-V3-jNrb)_L_4X*M>T^5H2vqaf8_j=pL9XyK#5iSo~ef?CJ3gq z|3$vn7^(ae<_TnV$eB&CJ0qUur-&HbN)9P*dJ!|e*8jt-o}NwVE>W>wK5yhjJGHdI zRnm@hK-_V{P^ncM5Os#UGk3$LcAlwq1^qF#EM~Zp#V1Q|k$*CQX-yT+Y9zT&DS7X< z(kla#CI0RDGoAdmn3D!l=}~I_@wS5Fd60Juvpt^SGXH1}7dS@fxkuxogjY$fi0gaw z&a`XJfFeEK&ZO6o`A)}g{hfT>`qYb}G|uEAmdX2_dpr^v@NGV&mk$xH3haWOFnk)B z!dyuBfrTc<3cqg+$>E0}fpGkb*vz-@?Fm@+N_~5M5Amt`;NPCPzUGjG{BC*0*dKSd@K!qWoP!zAX5)lXl$& zT*r>GbTYFfaWjZ!T6#xx!iE5yl%$|H5T{~6${aTR9fs3B zAu&&xOB#G@YnG9xLM)GjsRm^dl!~`B3!h94HY`0|8K2=I)k)aOm<20Sz>wn`3XBO7 zO|+ZXYK=Q6P3Q;(PvoWqzYl&mZf9Q~Vs~~X#46bU1g=utN6?0*GnkKY5Pc`HHRkAv zk9W(ttughieW$zrD;)&RERQb_ z60Db&_K1T>$8m>|2tisN?n)n>^@!J@PPlX9ID$}|mihXZr%0Da5r-xA_5^ZygO#n}_!cGf(Xl8bbR)ew^3U%4lY z78c2%;_V`eWR5Xloyua4#2D!ala95eW$ggvV>3$QC#7O1DZ^e2(4>4TS&Q^vIsq%p z&1P=;#@6cOKwsOrXtXMVcA@y?MYU$ajeZ7wUw^uZs&5J=pXS;^z(B|URxT~Jp4^!Z zdVZVQ>dZfervAHl9MO-PZ)LUm)_Tfa3NqZrk zk4WBxO#_++g|G)21lrh>+T)8NWoIq6;yQB$F(|o*Lae&rDLYc}9@oDSGB9^5qeY&U ziLu7^7Rh3i!};wkl4YZe2W2M-tX*+6!-gU5XHSb>mS;_&H|Q?cu8o`OM#0KXYte=J zSFEwsFC>$>_9zCGc(el#y=8-B>mR)$(>z2Tk2)QK4{l%>Wm^jJi~^?zn_tS?Zq>Gj zODjtA>P-b5bQaZ{9&+u=bP|&FF-FJ%CV-rQRvWaYa^^#By9?eq&U{GOyD$_(TR5|s zUV}DJ!#a2*=u|C7(JU|_Q+z85dh;hpW&G)m(V8=k?oW<<>1pYtw+Od=bX=@y>m6BO z5dVsxRK>3>vhJz?U-7Rrfr>@ofJ_f-mNU%0tz|+ijJfP9@f{5Ap1A+qUcGkZj zC5qJ4Ur5O)Vo;Nr^{UXA@e7HPa7jlMC0GUaPN`%B*`0=LB6Y`cP6?CRB*l6 zUYoO6)RtsFZ?+zUEK@$A*33tvzoXbY$AxB*1C0PM13)s@R=D7}jlHk_%j95{gM>*E z-YwDL-7$7POvZZoJ&ZfHWdcciLxCWCW&tyW%$iE~XN!*)kon`KB^ncg9_xC{-KN-G zxT`1OqG}AvN4;Gl8aB4ZbaP@FBY1ddMvhqSz2wy4e-qGak zs@%bx@_~I#$c5otE$?F;-$u2dlcn421JFn&mO;p26N~;Eb=5yc4r@s5uOsAZ{%co? z2$CEq;dB&eXHl|W5wy_ulalp9ow6QcWs&S?)t8XvSjAUXq)#lBPAi8~P=kf!x zoDvh=`O>DKm&I?LO@`b@5HYTHk?0>01dSjALT@~}EEJ67;6s+af(vp{KnG?syS{V@ zD~SUURLSD+{FOKxs{>SaB6au_LaKq^En z{d3}}PNYJ9JB^F zLO$1W@u$Fs1wMOOvUnnZ=rAy5moa-G1tb#eTRVN3xa2LJ!X~85G)P3nPp$&nP{{C& z?&YF(f~#!VrKOd9NN04XzEv<=ZHbDQdXX(+omz_H*$o3+aOp6DmKeIosI*;}@L!XT@o-xuWcB+E^< z>52J-0#ws_H{Qywb4K8xftSRdDMF)fW((x({^lgrOn!yEIY?Y%@c^WR%pO>+yd6w# z#-}_jlo0a%RboR3q(J-z(rfH^!>z|1&%5^7)1I}StZv3t)mf=r+x76os?SaO2z z-Yyi2j9&v;IZvemSqp{sGV*O0yYQ+!*_o;&86rI`5MYxMZ8CO+jUl8A7Id>9!8y#z zd~n7e-olC^KT3yX{grRX=&d?O1V`H&dOyN;u*v>dv4ehJ+ua;1Q)r863DS=O=w?3Y z-*plMQ|2AScY^^AJ;rp*#$+TYJYOO9(d;5Zg*<~QvbP(Fvd#_Ecj4XGcb~Ja?3UyA z#3b5kQUOd?p=UzKahlH3s9SlS#Ux>)QZS_X(TTij`DZU~V9T9DYSpPC2kLd}_I!dk z%S&_HpS{?s(d}zj+Ip6qNgb06En`o!q@|d&S(To`v=Qlii+yEX1*vI4U4npKrq~1% zR3M-m#A3+hr`&uMlq^0114?Y44Rjs*M=qQxo%5TYS(q%2zUg7IQvmTtkF)!V&<6FB?+dtWM57q#7k6N%%RurmLgwkE zjG(WT4r8>=R+!xNcs^G-{2emyP%;D5R5$gO$nrwi0l{j011X#Evz!H3w0E8CkR?ev zPB*^Cn#uTJfrpVBteN>$W_zsB3`l)J^FUMW)J1lc;s8#UD$mr4 BzSQ;6MQ{jG` zd@7X9eyH&AdgJ4zl6yCe7ME!;5sF{Z4hhxw<SA2`a#)7iG9Ms>6 z?uZv&woI!K*`UPU_5$;+Pj7Cbla-CaNtBcHY^-WXi#Y91Sj0`(#ZUBTgw(+et8u%& zHrjfULDF#|W$!bry50PRcwvl=C5zt$A<>TPNXU)8R%;oQ~eCmj}}W7$|8M!#((jgQ{s!M5>LJ%xDClUEAH$xrn#xR z>-F7o&fWZ{)UB=hR#E!a8L3;QzV%l&<0WzFJX(cES~V?AyGYI$q2Bz-9ErW2_<~dQ zi&CEXt$1O%e*Tx7JGZ)0x61Ube@Wl^e(F}ezI9Lf*8QnlhZ*x^@lBksmL!q~n!HZX z&q1m^=*pbaD2^OXx>x6rj%-Q>BFa!)m8nAXH6}cv)M2 z9%p@43YfF}QfS~pUiTooq{#6(crK|Ha`hwa`Pp_N#6D+nwU8`}kQZD%Z@qGsO(@kp zzI%TxD8ouQjo_xJ$ex(!OOMrz`xFMR;_iwbkJn?m{MH&#>RygSf^+tYMDsfbh9yRtEGJV34v5H!Rnbu84paXg z*hhzaKid}e?heK04zfE&$Ydtgm{nli*Dr|b4jD@V_O?bxagZGP!$h(3KUqn{@*B)s z))Lj0UZS^^;R0zMMQo->{w%2+J@@LKs`!_s-(3_>%~xI~RB{GIlv37uiIr(Atl}XW3P&A(&?K1n6z~G}ZEsKA3{eW>Uyr z^($coUtKR-@~QarB32C8_RS~~HWmST^;6IlKc9m>-IiOhmNDHtC= zfsM{};c9BP4ogoN(s9Vg?1~Zdeag$?$~|hLw(TNfR!fUK*nNEYAsid_ejmNTFU zHf6A93(jN)Mz4sKMYh$r{{6~V!w;*aSuw1Q=}m6@6}`vJ?h$| zx669odKBtJ6W+)QXm-#dmR-J+m%OOz!^*ohDEr zHkE_xtmGgtg1ibYv!xFkubZ3FGH6gjghg5gA%KvgNTSYeF+fu%JTJKv1@@A<$_{%` zog{E4PKP;4ggNADZ7F+YP)-~{oH6XkR0_SGzXQPCGR^8&Eq|~#t(?2q`hR$s8CUDe zTllTsWt#x((U3Kqw*4W`TT4#STy<`$4yi9rMW1mh3fdLveNXjn589N;sWBb8kq90R zR?DC4^`A55gK|>Tgu~#;D)V~Wl&vqhZc3G}?Z4%gIb(iy5O+o-%Jv_Z5ox+?ne{2$ zZ=xembWRXA&3ubsWhtwbMicsCa$tY&J<*T**7o4dG?n)r#i7Xxk$TcT{v2L>Tv8XA&@~wQu$&n)Tx&SeT4&aTW$OIMEF{p z@O#0zgKz?}J=)*8MM|4q;iK!qW}va2gAuZ&2c|@VeIEi1a^*bVTD6S5lhOUmS2xQD zYu*1FCbSyF@{mHK(Z>U)QD0AB1+?vT9o&3ncoY9AfwwDJ;IBEBwPKADUTqz(i89l8 z8Gv^^wPr8f(=HwE<1%H?f^4+Jx2SHG|NH zFisg8FQaV8$+VZ4i>>tp`@m%hIP5EWK18;3YLwuapD9ylw3b{M*&##rFp;wgaq6ld zL60?3I)I>+=^Le!2e2R5FV;qC?BbO1FyL$LqbytV$i9I0|4NQOefs{cJhE_O_mI@r z`a>?sy$4$5;PB9;_h5e(Y1o15!Rh#KG3sE)(Ok~X>sjattP+LWYHgAydf`-dy_9b!7^e03VmB78j7kzopG-0@P90Jji2SrX>gdKo=OP zfl=t|j1klEESwW$LcfghKmvIVyG2J^XF)2OjLOiWxZK>l74FD*m_uRcaUtG^6XKZ> zA}ZPrK@+Jxm8AZ9vtL3)9}BVfL5O8qXHqQcBLF1CB-QrvM7;oE{SQpyw5jjB3%6!9 zIABsDLa~qfG{xem2*oxQB}Ro>UKU*#GOzZl>N8%uj=Y2kyms>8-}8FhA)^@I zc>ZrW#yR{GKUT|SGMG6oyUTCS3s1wxo2)NuNw^vCV5et*CRsfE1$G1VU?npxrLU)z zKCU5?RB8JmSKo*~!q(Z^h9lP#dv0^`S>=6E#s18Elx=0Ft)U#^01q^XeL{YR)8x_O z3$nA8ng80gyyLZ9G*PnnS#5Xq6Eyh^SFKghhKf(?yGl642|`H9Z7Lz*xFLvJs4Ve( zU@cpp>1aj3_qE$1g^Z$CKQ|9c}jnU!kJuLf^ zHcxwo%bw^M>%L!5w@~6U>+OS7754aBZE%GhCyM%##iu+k<2_mfltV8AKc-{`eZqgf z*88L+ENeSrfZQbD$Q(35{f@|$R#Ty-UFL`Kr(csErVDmSmkTSnq!k1M=6YQN6A@#g zRu;iKh%+XrvC30-|HxHr5}Co?oW%NZZ*)8Kj8qsY#QZ-5N)sOwek`>1aqQAB?ydF? zw$uF2uZBb-=oJS6qshN#w2qE|fZo?vHdLyp$JWu?d5R2z2R2!k(gKwXJgza9KmQ7_ zK>t5$HGxay>m&)-=GxV>WSeB{>dEX@hg}`TJ=`CK6Sr9b7hv|SVk>o`ZD!$6fCX}U zk%aYYjig&-vU^%-UXt$>iSMANP`Ly-= z7KxJUX)=3)5*m&cVI+`%QpgBpBEp3vMFDB~y^c=n;1oW+-ch!?Z)v`4c>YoF6FEyJ z#fTN`3srYS#wUv(d(N%c9uk6d*auqdAJW^~?vhTNhKiC0`>daA5>la1YNV6Jx2NjJ zzLNru1DhV|vAu*AcSP89xQ19jmt^G^u5A>;>yik5DfwNr7#+1vlf4K(bTuI@Ti&e` z;g=)&bTgX&sF$^^HB`_QohH4lqE_jQ-!?xkryZ+HPlHRhqEOgl&$|70VD}4ub|;Jf zRo_U-+M~sXL??a0)k!zm?LdNK=+PgqqNPS!N|X!HkF~-Ga6y{vL?R7<34u)}gA3P5 z;k%^cdhZMY>x^T8HS`!@1+H*61W|hPa9Bc8`o*Aaos9kCfazf7 zW;{St3J~98vI8q+0HcIa&i~k_ZN^z_oj2mxj5AhJ6=9r1Pw)%ld`cz|CkW#VN2}6b zx*ZVfMUnxOPK~`zSg4aEm3j$LJPjXMcj;jnOnio?!K!2vf?i(B!t>jL@rD9nqQba- z#-6v|WgXC)!DyfVW4yR2#ZE6G#jNja6n4_fW?XjKFR1PMO?t$825vyF+NFH0bF{{V zTL)yL+hp;#bjcJ6xS(r#8z@KeWe~Vcf4t;JMt9&$ZYGNxv!Bl*A;G?1OwWpx?26n9 z${%a6Q9jGsz>F%g>H-<823uMowK4($4{MASSS`QOco#G`M3K*!_x1~+DsC9~x(GGI z(MkF;cysTl zMcHk=aN#^mfK_t{(J@K0;NMf09+X}XXL}>zKiO-R=w0C@QnsbD@kR`@c8J_zPupIW zg3cJrDBxeD)@z6X3M+ah?-9lOZyd=2*0ye&asEW2iE9Jx=ocg->)=F_+rBk==?}fM zUhSjfvM9-3kR;QilJ6^v4!I;Vq_QILEcRW*NIF~A0-gayoQeS+8_et6JH(d44rgJ` z|03_0x9I8|fFeavkn&S9%4z91r4jFr&ybZyoT7N<@mU%WJ7T|y z-1_X*GO0QlmsOc9BI-|CIVQD#1{t&Z-O?$rPPqStU4P1A7Ro;E{tx!n52^gGt`p6- z3cWb~I>t687^%GWJ2RIR;$EfoCsqV+0#mD|v+9jm2H7kY!B;42;>cnuGUt{gSoT*< zkTHn1@bD)zvh#!9|!X`?y{0VQ!V2kZe*&kTZH6CGQROpWFLN8C+T~l zE3V$#`C6>wo%qmuyIT%*;L0sOqvcRDv4Q2j$_~mR&PWN2b(D0kW7WS$3K8yiZWcQ? zPwQSM3L-_ELhr~QHanu(uG;F3IZp|GehJg!Ez@Ta1218okO>tm-*!IS%!i_d?wyac^q*V}bqyN&T9$158jT6tth?0@$8iT3}= z#gB}K&fkSf^M{{<5m5P*4j;0X6+1_$I9dEnoSoYEbUfH?I-+QlkT1(Mb62wXE`DQG zXGf~r_N=%_zwcI6ZzC?gW&|5^pS{}m>y63cKT`(6Dv_|G-iS{Em23q=Y~c4yTIFjg zl6n#Dp&!9R47Gw_(l5ew23Y$~0_!zpX;|fT$l^~K4^})tDzkn)r7*f8)MP#*lKt+| zhGcOwmauJsM7{|3i{;7wWL&I`&q();>BX5+aKT`*_}@5?&egQ-U%_iq1^VjV$P^8F zWRjkjkpB7X{vpth>>qPEvG@~h>&#m#K(*Uuj3<|gZ#L*!ORy^$B{Z5ceDJPAxD2`UZ~=_Ve*w1VCa^{o(LEn6zNts^f}ZOdk;W3?jm3|)pJJvQsT z)$S8g06>?X&5J;Xh_l7(td}6v@g$GP;_A*N5+c=GkqCt$fphYQ*C3k2&Sq60 ztDsCnh862n7C>1)lDLHVO?@f;fb^Cf?8JMr`cV$4JC_$Ym&KACCh?Bcl6E03NGLVu z8OsO(EAzoJ8e{Rqz>UC){ie@xzXrrD%9?+W{c@yzr%lU0B6Z9g&I+XAX%cAK<8ED% ziWO)Gv?R+4K7&LjUIa1Qbz7!X%EsRe4Dru5sLw^U{?)#-Y_ShLEXCIOs^A5uT4oXL z8U_xbl@xI{{)B`O?kA%m3UmxY7pLsNP%0KZF8g2W9wXSr&eY?h?KhoMz&jMNJ*@j= z7$M^`sT0C;UTcBF<@RvkCP5Z|LciL8!bSr^-Uru`L{mcvd#OTHpf&J)=fxNhrMxjx zh{j8{2TDxu*qcR?IrZT(q~5S@*Y>Z}mZozda`i5BF1?4&^Jl^tLhM@k3Vbrz#sYqi zy(c-;$ZYKMLXcH5!HeJs=)Jso)vL~544HG`*{vb-8f?dRtQQGqK%)c>z9NVZktjLx zUy{jz`)N)%vmP{t5S$DarR6OUor_6XBV zI^+HJuXB-7tWw-;RsK{kC9|zk2{bS3-I-%TST&LqbbFCJ4k;|w0dl{1@WMY%qXUl- z?-l^v^M{z+k#n2B9hY+VD(6kvQC@>$ztHb~P}3V*$E<<;L8iq0>{%!~v%!vUI?o?k z<-Bt{($KmiJzK?8Uxhp``pLF8R@txSj+}#n;(b%H#;=bTa*u3Uo%Me0ibQ|1cw{G( z)2B^fCd{lo>p_aU(am!r(++LbR|G91cUdxjMPZKqeUUpJ;flozavo0UBO=ovk;bQXIo|LzRltIH#B+{ZEp;FS<5*K+}7|qs)4X~u+H8< zX9iO4;E(pZnIxr0kQYe$T{rM z@NAJzrzj^F!W)Np1BCPb2ePMm{{#HUes0n7*x{n3mpb_&_WK^fJw*c2Nb3tunc{I1nr2mI7!|1Mc8kW{bAf#%mg_J0Fo+Y{&;#M^UuCh>=$Yo0*sP`aWr(^qzE?f0)IU-h-KkwN7$}A3%nxu9nyt~wb zgv3D#)uN{xF<_zizT^(~Buh5%GTmpbv$Tpoj%*N+GvtHWuV3-BG0QR8WWDyC9N@AX z6D+D)ucQ>(!UX@S9?QggCkup6uDstpvRsH6&3y~oqks&42 zW6Ka*goJohxgunKwg|ycvqgv`zwsVBL`ZyAnU*0fU?x|Fv_o>4GNk?lGNk_7whZ}L z?67xfHTJ8>4>lWj<;Qu>edLF{OF3J9tdho%9|v9ep;EHrO=L&CEjvQTl^w^V_iTBw z`h1V)acM70Ufcr)KA^mC+H>YNm{Z>!WX?i(YB#Z7ai}p`u*NrAlmAudHXNUg0NH8F z5Z4CLM$C?sca10eY^}GSl*I*ITIy&q6YDNhi~5i;o2~C_8rGa7Apu+S*$ki4|Fpc3 z9=q~JI(~$_8Tpf3d4o9r;PU1K@x*l^#42~B_D^B{L#198g}mhb4oI|MnaUdy&a094 zw#-2rGHJm@v|*R44aK*iIP1+tSPs z+V5)*+mt(AwBJWE>VK{qF6yX;bgp{bf{Pl0RDzVMOcaG zj0Z7N+{dZG0>a~5llwX!$eBUWDQf?Cd`u%IrUB6Skn#SY_=w?si8TN|yeGhi2#F7Z zj}$Uw{BrOi@P6R;89nz+4eL?g-&C}8EZRJn>E z*C}Uj{FQ<@aOG8itSQRO|Nb}!Wc~ipAR(w-I;_(TcF|66@rB7 zhcptT#}W~c{>NTlfo1X|wvlmy>kAQn7-omrY=Ej6iwf!Zbm1Kt9D1h*vC8^s0hMuz zzrtz-oh!qB6R4j}jElld(Bs0IqIr1gW1YTGS*kCe5P)yb$p6de&yB0>{ve;-{!rHK z%tGnRtzV;s*i$RgZ*_Az^I}n|GxLSYAU22Wpf`ZgKwC-Z7s7(np)iKwy8sN}0RYv2 zR|w!0dLZV}85xNA((Hk++RZ{mw09)`+)Taihkty65%MQe)R@UXA91_TTG}k|Kw=+| ze+FTK56M4K-CbH#8T{jsZXb_-ZuX_H4n-YR;=fte@sA$a` z*16DSowPlW=AV77?}vZh!Sx0SNAl0-$D}&{zWL`?A%OSGKQAGPr91jEeBQa4knK)n z>kQfNn(&x)F{U2Y275W#cIhd?DEV6aN%YL!<0+jDDBP%dB)|g_!j}A7ciuIf4+Q&v zD0qH^jdmnFnypU&&m+R_x$vAIUr=1<@hJA{LU^R$hHAtS=Y)^_{Yb6|0`q2Kpq92&mI*X&Biih z$AAa=&B(5ic*6@<%nijC{RdvhE)9SFCwPB=miu+5usGSc%ko`Cs>Fi5^~TbE!aBt1 zn~b{A?RDgyQi*Kz7AMp5MyEU zb5Dj2yx?n{rfG0C)qA&w3{~bncwT6#Ct7ygr^$BH6`Vx{yftWnOVWUXjMrwP9n4f3?b6 z!CgCzlXyJIsOD@Y!|FhIyUci+b?hxCHljI7%m#Kd)b7hKMZX)<{h^k>NkdB~gyK9> zeQ?Rip?F(n>PzpEp}d=O$RljNWYb3bbS401;#f|n&M$2X1L``uB)e?)@hgjpRyKcW zl;FSCl${2r7e&sj6Zg2^dQv7POrF_gwd!eDQfE(1FnxW+b7W;$0j5j1EhVq8ZzCO) zyg0zD+opOwf;u6KGqe0dJk#DheM^&6^bM-}vjUU-x;Sh81rBySV8BONck4hv{V8I0 z1lp*tYvR4Jm%Sluodr+R^y&q^$pg?2gKk*JiTh78EN`Y_Kf-jq6Aj_4|Ck`A=COB0FFn`~Bu_ z{GY&+Haea2Y3{kk$AMib8%ONdvDnhUbfo>xHT<(+fDiu-?ZJ5|yVkX6|Kata#Z$lu zzGVKx+mCX$-2~3H4jm~wKzj+A&B<~X#NTM!fzD=0+#GcF4$OnBNs`Q1mh>S#9x@e- zdI*;N`cjB9v|u-}#h{*wj;}<)k=DeT1(unWZ)Tar#Gdj#mOt*?8)8eO!Pz5*_A$qZ zF5bin1#~gl4m@ynXzGyY8R7ChL9?3&1_=#rRXbl*2c8RcCV1+sOLmEQh&sWXlLP@g z1+ik@PydHY|8qM}fBv!kr{}ft>(&2S?`6E*nIah-u}8ND*X#MK@=aRyf_%YwmO(_C zYebL2CN@9n2eCspxj59%i!BxYaL>y&=NoBEOeWoa+6=Q>#SYu9TMeDX!IlqNUiA(U zTfqg_&ZqpU)|PCns;4U@UV?3NSgigkawW{&S@y>A0$yoE`h)#mX$~S^kgre4SE*i1 zCcEN^bMlD2z*6PS#vK?g%?^m~(ldP?LWpgpSJ|8L54_DCDL-GAR|RV7Y`^kN=gZ)e zS#qOO@?=Vq@l5$AY*6RaMP<>E9F-|! zQAJ`T=M~YRoP#%3lanv5A16(aZr!14o(&)hSTV!wese4v#`@ZRs8y}_Ok|Q!fy`K> z9Jv3@@$zUu3*Ps5J)#$KzMt`GqU47juT#P0k>fQ{H=Y`=2uktq7_STU-vU;#*7$nXGpUkm9VjQi7%fyttfPIEWqns#}!;>OT!LFn)b(DzLfQ2$N`d=URDQoggUKZgR{;W&t5XxVi#_P5taC#egTX^&B=vQjMbU>-X`P6w(a>Ql9t3|Uc z^5fEGYV?Q7XH}`{O}_iZS90vP29OR-m(Nk`31uHk4usKwoZvb*8jplX*MO^Dz3CpH zw{K371K#pbqA!S1U^EmFw~^b%9Pk?A3Tp*LIB8uhoei4>6d{C8SxSP#jnQ*$%xScR z%e#Z-u`ze_N_G9{XB2%3om=Qv0Heb%NFB*`zqo*;594LpX3puV9P>+-=vMcWFabx6 z6pM|U#1CXlC`j*i$ROtd_6U8Ko=HxH#A87iK~nCXM-G@hMb#$BHQ^AyoJA{_}*7$4lLQI(jWxugGfm*{k0Zktq{ySMN`hO1m z5?T6;u!=`Wo?xHLJKc&|-l!|ysVlCOiY2q~`m8N7kAzQe zjkQm&Pm-J11lxi-_dFHxmkn1kHPswbuUc1S3My>DX zdC_4y_}gs^N$pf-W9XlKU)v1fcya0Z|K3I->%rKe{N-gc&HKMxvV-8UcT+I= zk_(5=hSNEX?9%{aR~=+4n9R>`o|I3}^cX`@qs@K+==0N7j-!boD~Cx#q>Sn!z2dKU zP`laX1-^kbw!3^6KiqGaFub7y=mtXEr zXU2lK_W#S?`+!GPT>ay-Asa#c=e8~B3kPUHwFQR;>ZarWnEvqk10R(Fb-W0} z?YsC427H{^=KSv)$W?Or+TGdTq&vLr)*TwgtLG=+ z9I?XZ)!I85J;Fv$TOsqo!wBdhg4XFd455!^(-S6mB^UAVP0dJee{gy=MgESLxkxNn z;m31M^cEnVe)5nWNRKkT=r(NFOTE5Jd-wEQG4uj8uSB;E59bY^>vf4(Xn%1daISrn zSc?C;rg`51j4m*YLLTAq--q1DMUJ>H2|@TE?%9nkO0eqQb}yb|#*5tNAOVM$x#uIK z$6&H)Nq-}igsvR586+<}LE7jFzjse0qIYy!@Bj$jNgRn8fa~Tx;g8}HEV`oG$9V-F z%Kj(N_ZVvGm6Q;_^Zc6&r=y5 za$XL!y|7v#XBpku>qB#r;&$hwGr99No6d`mZ(P4Mh?8-O@Dc^AOSIHuh|6S{70iyP z3&Xao1X`ktrdA0SIl4Fi(A)k+nwJYZmC9G zBFfn3+=S%Vk`DK8_9sw(q+q1^4Hdf}sWFG1kEGe>1fG4S^L!_M{`t^W=#(=X`LN>q zJbMdZ)$F}<{V+43gZdv(8K_2N#)I3Wp`^Ze))#FR+tVif3pwYfIjGrP_%y*b+^R-r zIQLc>ew-)MLoJw#;MnN5A06jkV&KfnUQ(e#2=MOZy(A`e0%k`bg?KOUirkJ6zukRG-SrM13EX`oh4&zu=So;vWGEw@S)S!@6$rZnD$G;c%u-WstI2sgz<^x^xp~N*v-~wifYBcYqrd>p?83? zgbckILT^7_DnP@ZAc7wPy{&QdUOEGdlM0=?v0vU)|;9=97B+fGuOG_q$O+(*njTt=YW7T z?RSLM5>2c4ubS2u>+f$l>D_(m@1gfXrnl!vNiX4T00`kpM5&Ro!1aB?HJ&)2-X8ix zXrR8HxPM=nsJt!!NlG=kyu~7=p_OO!Hw(mK?O7$HM{Ck&BoA5Nx+zPvLES>{q5o#l z+=v4dhN!Am>pA?t5;9KA-NH&xTSd+_=z?PZ=%3OS!a-HJj7^sG&E zaJ-Ozm<|jGId{GhWA{O0Uibi6tliUx4D}#jTY1$6(rOJgxWHUr>K3DzY z_}wCeXKBJG48H$Z`8|k7m*U!MmXLk%n=FPbO8+9`zn%Vm07xF2HT}NVbh8ab9#;P8 z8Lv40YlZ#hEqj`<_~1Y z(Q6cXQ#HL8UmG%i2dGNX{xC!6?eF0FUihaW)7u(H?*+uDzE^8{w+x;?YkSXSnwI`` zwU9kQlkNJqW^*X|WZL@x<&D)}2=Bj<>wiBLJh=9;^4rQZE&T2Wfc0OK?fBE@=Qlf! z-$o%kRTDlH=jDG6exLt@WMJWUs*v4x1B+qjzlP4Q_UEnyWf@DK#4X97LDRgI>H*OX z+Mi?p@WdEBB2Dy;*Ys`@9tTS=+WwR;*8W0tf4kOyAvy&AdTSiL7Z79p*Yx)NQCE5h z^z!29Z4r92G`+v$Fr`7Zx2bO!S)$cFUC6z5Jy*B+6-{pN{`z-7P1^S@+)`D)rD+9U z*0cuO9@hR-08(-Kc!SVBL(|R?+Q(Yvr+C1b=BKfx2b%7YKbG5NTZ!oJk3Ek=uHwp@ zD&_49v0SS0q$`Q}3zuj0&-^Gf-{x=r7%-CI`MTU)FX?hgU84S30iByONPm-_$ie2S z`3O0_RdIw0?YG`BOUiu_-{63s+l%wAt{Nl^c2{}(hL3;kmM^%^^ zY^{_(0p%Bm+~F6n+kpkns7@IdV&+^eRDmFrKNb@+Ic!3$G)JCcxmCvM|(Q- z4G?`4r?%e@d=Ty5X;Q{7>Fwx{b2`ePHXC&RX8V14j2@9DnN?_d*JEvT==3JV=n-k6 zH%`-g8)wlEo!*oGjMkS(6TQ9k_H@WujObA48GG9Vicx#}I3h%|PSfla84kwY8hY7Q zdYgpav6^1f`8rQ_ueg%42=|7>@|b_;ikx@sBNB zr^!;MMEaXcLxybyndKGVAIn0{?|?pkWk|0ZnEv=Gz*L{>bXi$cU-Xp%$dCFXtySki ze!M+X9NN$FBVl@3h`!L%u8iSYlt?bf$eAUw_>Z)&2WK5fUh(BazvgrK_=;7$ zeqAl)pP9twjf4=oTV8%f4^?{LChzj_5fOa@ExLQneWXKvM=JbYt%RAX-)QX^q<&z z>NP0D=-8YSB7!K8y!b1j~&5dK;x!wdM!_RQn%X&9y&HxWL!F(;t#i&Ly)l45$7} zWM@`C$zXY^3H4kKQoIR^N*ZPPwvb<}$-nYDUCEO{{$U~SF_pyLj2=Wo`{Qd!RpJMs z5{%y^*#AA8P`*>8=zU*he>&qS?N6U$@uzW0d@CP0%8z`256GEL#>r+rSlF%8n;{^K z<^=Z;UpYL8d&TW8qfxw?L?3|YOQND5XgmPFCxTx(M4H~Gx9CGHz|;~GG{3zN75V=Q zE&R?FQDKu>^Cf)ZrP9ZlHSnpqV{Si_#l-U5_Etjv_5)1(ZN7@-S^dAqpV%X)*x#|2 zi~h?G<%u~uL)hy53M=sIh>|%v2^#iW(XeA7Hg?lOFD6znOrJ`q3F)NVSntu#v*5v2 z>-@V6&zLY1IBF2>_i?O0-vuzq=R(cMR$-+7c#bCl3qp@;PX34dMNi22D4uq=c667p zw+oLPVtDw$J}sxIkkex#r?H>4`eGe_DsA-drl;#s{Z*R&M~G5842ixOFH?!O*)Ccs zgd@!yi#O)K42I2oul+pgx=kn-YQ{qF!U1i7fAYIP*1QPa~r@Ukh66; zIo(J&HF!7KGriXM)_FJFh_5i@$EK%x&A00>lsat zX?u2x#hy*WdPX&TY%;m{Y&ETJXkVLbEv%RM*0eXl+6Ece1nmKSp2W^;0)9pdKlg8D zezrgUpXbNuuMZr={JonIBS9=e!RUFLQLu5C2D#sP-uZ|oJ7oV>=AS>J z{amYodkQ{E*G&}cdIS<-CihK)*=F8F6>kz8*G}Z42)hQT$9+%-~Oz8KYvG#A( zkAv9ex6scK`Y$!GB);>A=6Y!KjXbu}BLq=_ynq-9s(fcQ2RNvb)y>$-_nck{6 zdUpuD^E5p_*d3(3x9WE?)3n+@A^Tx~tG)GMO?D{u!?gF#SCKa=&wnDu_1AYo-7GpNMA*{Y66muXS1;zx|x_`}tGbP*Q^Znf(Bhm^Mp! zvJD*P&kb?C(6mPu*UdP*RZ`GsUC`N|vEQ7)-gHxzX#2W_-b1x4sB3BOd%5^o@LXVNX0Njuat$_eK^6 zed>)EAL{s4N&D=U0qAbTNgk^-*)f#-MdXoeqkq28KeB<@o+oU_&96R7-bNlfvG*R8 z#|MZI6n!h;xXPOrKwpz#z*JKX}n_uYp z1#Yi9v9m4h^)YUV^&ei&qJ8^C&3dBtf>>?4X$j|F-jMSg5KUk|Z;-Oi&}D7EUzcTH z?<}SRS7S?oczJQ~>zypmczX_h9zd-U@bf9aRQI3MaNSGp`(NQFfc3w`{L~6R6TlDV z%iW05`rjJMe0e3U`|SYRB&3q)n=7sioC2Rg=ZmyX--%RS!t}Lcsgf53Y0;1-)Oux* z*PRGn!tpXpvy0ntlsG<;mr)d2kFyCHJVl{5cp)=^Ji-6ve@H0Ly8tG6URcfi@A&oq zHT=i+GwyzYSl9b-W{LU(z=-efuVX#%A!^*U=<_;5+0TgAzcQ@vH?A7ozF@C-;}*`9=F~!_I4h{<0rnlGEn3+`hEm{C|ah(tH!W{CV2{w)%%th2MQu z+^$W}Y59KP{qcBtq2!BEasqkX1~AF%Y+b(hUS0nGu)Qn6`WI@NfS=LA&;6C$emlVr zwI4pa8-2d$OqdGTkgZ^xEDUx!@JtBI46V=c!d&b0W~7Np&aqgP&)jMlXg?$;qz`FRFV;-{&C#rKhjZ@Pt_EaFF=&tlmyD@WlXfqg~!X$i|eU&=qS zhRc5fQIf}hx_s!*wjR?SanGmi2blZ6W*|ox@LKc-$KPlTvA$Os9i5^B(xA7f638cj z=O3Rj?Gwk}EaC4(dn}J0FDzos169x4MIl%Zy9I`F}Z3k*=hR@G8dWwF@de~g> zV>M6LAWq`U2D?}lo_)7A4{`qdp!Q>|&!axGpPDXyd>bzXJzS>!9H7wnb^=!3%=mUp z^m$a<+fSC~Z?QdQJ&K$Avzbqk2($c#Xr4RNDb4Z zo)S`t{L@@ocC&e?3G@g3>rPPVZ*Tq%C~>{r&+2o}&lA`iF*jrm6Iz?kw_hQ%|D8jcWi?4)=GIU zU0%;GK7V;eKX+lTI;NjaDd4WlxIz;s5glSM`l-*SvhD|aF#dx;g7Nk?fJts=>tdRI zu8a91>qV^3Vowe}Np-T!7iUR%FJ8*kYW|rnZz$`7IHU&K-KEwuYp6@YJl#vBVY3>i zoOT71@F*PjHR1<{gKe1Z;SX}tagpza{zT`OjYW5iJ(P?U-PJgf;WQAT?{nwnKjeUH z(8m|iDJiE-1H6&eUnb)w;{Li1Uj}SDZ8#!_r{EpPCm3}c&R&SPCm}85e49bRK4dwB z+lYG!gP+ylV(@#S)E-vOy^{AA9tqMjEe|mIWT4Z_p6+2lJB0l%)Ght{oaCUScISzY zC#gfwJQWU4qvBny9fTE{LS@h?A$SDi-*o&*E+6Vh3q5!S14lc%?$224x_^{9=~;Zn zprtDbvUnS#5FRi17d{w{!_i)8e0p~dXhqz2F&76&qO-wGK^w-NYMd&xsR$qUT@Sh9 zw8*yg9$d&4X&m8eo3a9>A5QkQ`OzlC1M1&Rq|QB?n8T55WTb6!XG^XByen4bv-%X~ zV$69rKlB>4)kw4zS}32Mu?aalK#-c$`&((F`8d+yl&b-4bfdg0xV$%(g2hv+aDN!* zdR&GOjX9pZ9)>Qc!U^H3jntJWcxuxiaaK82u`s zLn7q2c@1kgZsNC;sXhIJeN^RDj6(?R&Z{0xQU-5t_;inMF`!Y49Sf4wV!omfJ{bN= z@Jn>eIGyB&vo`(^ac6@LweL-g9c`qSrIALkWVw+V`Pd6mV6^7`w(N-eK~C&0#*-k! zp7!nQLPa7z*J7M=^Bi**rYeprYG3JChO>M_kMj{Yn~{kcsss&-!=}HIQPB1koDcOC z53XoKUk>JD4GkYXT>%ntuPLNDQLXU?iD|TG3mbda{WxKn^<+14k|BC%6&r>dDG_{q z2}FO#qThp;L?9S^k@a``=;dOTC7$$}QAwyM#T+PO=vT+l|21Mnf3?tmZ1D66)k2@_ z33PtL^sE8w36xt@g6zp(C_D8~|H$x}O{2owN)&01Mxbq62stX^${bAVZdl3SA zJU)2^c*OnW64KXO8Or+lZO-%$24Abe_)LRyvFxnc?aAeg{uI#Jo*V1SwJqrwz?i_mk)C?Um^`&R8`L81 za#TmYta}P|9a7Iz3dxW_pg6vh4GN5j7?QCloAt{BPVWHM5%+Me1l~(x?~J{e8iaF~jlqe2sf6<3=0g6={$jrq71st87p8p*gSvWI!VB<|R}F+m4n!jyU6S zGiOa`Jmw>p7>|>rspx|AdVF}&B;9BS73G+63`!tn^%l`_%&!C+Ob=XLFz+j#MjXZX z7{>JI_wf?<$3~3aIM5nZa^gtm!_)Bh8Qwsp?!bA}f!lY3J=9?b0_nhozR=UfZ8;J5 zhJ2E5gYLngieqr#w5$a5@GyKc1|3@ljs}p@9zqeS%;n`h66FK0X(1 zjx`7N5pXzL`Ulwg#`q6(euomLfYV?Khm^TPKfi-Awx{oByd1^@hP4Lau~}LV3LQjc z$~cr-6ndXj;9UAv2fTwbTE60OP}9B;aVLWYc%M#Zg_AVU&XL1>ggAVN7jdrQLQU;M zMb!QQ+ssY-YiZiYn1utVi>s9uC5th_t%^DmW!&0VP4}R^&!YU@Mf>Q9OdN8z2Opj} zQsi3vbi{ovXpo-)dBCB#5Do<%`hAQYiZ3A-wX3Hn^mcJuZgE=$+?z;np%|QJBJNC0 zm-2(ACaC3NGcq{H7|zQJToxWCFUDxQ3PT61OGOqw>g0j6zVr3P{@c2ogHr9Y8*Wa|ubwY8c^nKmgbOf_TFByR|aB%E~Zh z7Z79&2^#C-pSRxAc=58MOc)x^jI${xlTDeay#mX5{E6*RaGk}xT)*23J|gaXZWhwP zARJ)~LU|E)`a)_=9C1$$Le#u`2uw0B_;xLhom}t1=rSExXmf30{`#P9a~x9Jo_;wZ z5%*K`smxE8#9v(>TG7H{n^xz znR)Lwru}h)_DA7ur@e%GZhwT<_MgY?&p_S&oM#;Jp#3$t3hn>M1G@d^S=t}EkiL!4 z-v*UJa^LaFcGTQp5Htp-m2$VIU&hD}1DQKOd~L$tAE)(Vu{Jq7fNVE81@js_VsgF# zDyYYRCg;nX?rMFXwXC1XS&jq!ekZVJP?u(OpjQb z3t(-YHr8erC&4bTwOL@SO+n1s{MgVv(vR+MST@G!h!xXDr;O=>F=KBE+8UsH70|ty zy;=W&);+d2@9d`LFUb>e8?Q#R$M`dPK0#XK$uEsr9Vp+sg>+)zTRah_LC)2BJ2lOg z=UlDdX7jTJ1sL-an_n~l%A;H@mw(_4GCvpNp5_%ijSKm;g}E7QvkyZH9I0G~%TFhAdk>{THZZg)2rT|;Y5oOfUE^7ye$7xINHdS#JMVw3oUsdyuzhh@_=s1N~9J=*Q2-%kEJ%8WLfHf8?zAE$p^We zb48o%X=$>lEMpoXT&>ptCF-H9(hUeZ&lm~kEU$w$K66MljxBg-|K#h)MROK-5)D}J zSdVtE$bt@_?boC2*Q4#%N8A3}_gUJ06e1CK?OY{T*`{ zqY)AK7sa%br~M1`?x$ECpci{t9d7$I=+h)1P93fXl-1!n+(g{(=8`%z;GWcBJwmMx zZGeZImvEfbVHKm_%V4htlQUa{^(7KcKGfj?QHRseurnDw1avqUMYLgv-aVO+ZM|DM z_G;3G_kTq!twbhh!ym$h>AGEC)7r2UB#-8C#J-o4xIZ_}A>X4ov@`<@H#%`p@ZqWW z3m!A@@@91ckK-AzCl;W{cpTrQri8b_UI%^r5{@>ELMy;Wb2);juS{y%(?9$rRsNzm zM1Aj}7oZd{2){?b^Ou!m?E;M+l)r?fW*QhMbo&l$-w|&h$$L=W8(e7w6neC3gkw{>6!mem&6H zzaZnTjR4QisQ1%4+$hT3p8jv_IY-=nATv(PzW~=b&niA1CRKN!{!@Zp{4p7#=(Oh^F3U8b8ai`Y5_Yhk3FAb?7 z=N}o$`u7rUBko)V@6lk)Xr1%FiK3|F=_nZ{Xt_2)uGUPBr{B)ml96p$XditOm;7A4 zN|1wb=+eueg&%wavzwPvwU+%6rgPbrgRa)!lQD&NwW_FPAC?&*T~Y-sApDIMx;*4I zMAuKi2=P=D`e%OV8ul*n2nV<9Xy1aT8%NA*N_DlC5kZ<6c%g7zjjwKrxLS(=!*^GI z`VrW6wPG6pu}?V`T-|>5hb>1^UDss;rzkX>^%G~)N8IP3HeBb?%wl$16YNek=CalB zK32iEtj5Sw4Zoxs1%1#s3dfos5AF`J8Espdcjyl&XIbbB(fG(L;!A5h<|U}Kp|}il zCl}8Qe?vuL+I0A;B1{+QrXfb)p(ugiHQ`Iw_6Sh75hPz0W`c5W}tf~;5&dGVd7XuFsMdy3piCfwVSmEWZhRGesOCAA? zR}X94XPocN&!?uv=n-k6H%!xOZ66vvvtO|bJ9*LVkQAYL_gTzi8PbTy0iORRhuUm^ zV10~6mXNdBO7A{Ih)2Jsx2t{V^sg%>Fdw7ybzZ&6X*9ui?x|)<7g$5HopUGpy^Ifp)3%SQZ=Y8#%&0wGOq>u5cE%rTUqEB=egt~kSo~fT!+H+?}=po0#d_M`@Y5y+=eMklG8IkafD1vjdvZ4r#pIA@JhXtN1Y7kp5&*~3e=9M8yYMIc}H z00i6F?deOAHT3uJhp=dD_U9o%KTVErvzKrL5AMKVhJW>MptPU%+kKzH_R*Zh)rwsM z)lrNGYO;5Kiraa32>k)NaTVjvhfm4T^N@JGC*r;zg7aPbFH$^L%kdx!c7FCVrMz3O znw;aKE%mNobU&|$^F`c`oIsMnapV!S=U(41h@tiIyn}plY~ulZLvq+FBY*yC4gkzC z`OA11EjyeBPKCuSt09$8dLD{`D|9jqtSui!g0A*!uR+4Xt6fTMP8xa07r`RIGrnO@ zgs$ID^x)SWpzoBE0B8vlA0{E5@wt#iBS=2CPA8G#>2Eywz21YTzq9%2Z&d1rJAntL zPo+w=pLXNppQ63G!?%#|(8`lh7b?rtrtb}1n^!O=+3u^@hf1Q3M*uzC7XB1v$@fk? zS~uxn3->S2q9NwDU(yAQ+y!SuHiVJ*<3)&&#J?9!2d9#*l_qMLwb2066L%6uo(jSP zfslP!%i6v8;uF5#*Oqo}`>1B1eDD)OIkOu*BDuTqmDq3=weQyh12yzUeX?r+V4v(t z+(6G~kWaQ2_q3n2T4VEm)|E)X^A8MRf2@cz{E)!zJ8+--3O>FvK0HR-7q73f` zlo@^kH}D6U;a}sP_FI3Uv(WPzPfjNqw=w|yqBDEI!cCm%9ZtJSr>(#o=9ft%m7<9I zM@aF7(r-pmn==<+a?bI1jr$*1QHH-v$xxgTAT^Un^@P`;Q^Fz_w56X60D^zti(KvL zCn5#|gAabpxQ=q+*3uqqG3M`b*|(87xeADbJe@cOA5+P4 zzQdl!D!%y_%~>#5ES!jp*wAb{ElVh%1927*$d0&A)C91;<1qv##0X&7cVXWhD5CAO zXDI*!Uh03ke5|A&W`UfK?62!PaPp$QgCqJh-+`w``G$GIy3fL2Xp;XeBv0(1DuADy zi2FM_GYEE=%q$-)w4p-8Pm|7EBbgx(;^+Jf-;FZ#N4I3s)dQ z>m%oCRIp!z1bp9!JEeA2)G3N1y3VXrb1=i2gh#!0`E^0d2$EuhM#6CEN@ zgTx0iGbt(E3yet5p_KsThyISK^R=J(7Zg9{cR5oN`J@?Jp-R1T=AmJQuvt%puY*4 zZBu@AR}%j3!ao||HGr_sg?tdfSdc54mlZs%DAW`4|DuwZ=LnAYaIuGk&f^%Q@!g`9 zGrLq#duCVT0qs8L&Mw2mtf9jY+&emH69)d*ZJaArP6rZAiiT z7eko8i#WqC0S+%hLxPL58S+y?bmtP{b2yv{-5@8lH#b5LY#j$+BTW&eFy{SK*uoLG zCv9Zk(hT#5sDDOgIDrwFF`WNL@c&5uPv?IZ|BvDSas2P*{|Wp*k^hh3|0(=`EdNjA z|Ks_82LGSH|1Ym~!9NVp10#}E7rEKA)|W#l{2(qq9=3Y|h=d;-%gsNWd->D! z0@FXJx4fK0xeH*t+ng`|5clvHVaU5+DVJmlO%xjmihS*i ze3#usIh;b|d7zhSX6tIcI)~y&^>{8uk0T#li{(Nnj4n(hk;KS$9ZOs)l;@Smts(iqExNz)jA0s@#kT{HQU9mG&O{xy@=;m(2FEqF3VZOZT=EK!ZA z_+*?9PzI0Qlm^niNH^X*phA3+t~Td7HqLaIFBcxBd>&xqpYqK^*l--d z@XbI^WKZY`3~FCOMwjX~o$00b6)mwVTu$bEtg;ETc$vuKuKI#QpLl zB6lfH(laNp;aS62X}{#;Us7qg@FTmZ$THq4`39w+ym_(ms;RuPSa}%hig7w$TzQux zTV!8LB7R0OX3sLy8a#Rx;; zAvo5($g!h98mL0Nao5fFLj?EZWxK!o5S$ugfD}3{W>*8 zd-`Ss;qjx{X3q@I)aV|huhi)$Pl%4|fMwz(arpN_`JGQ8J$w>s=iPP$V##Y+_&Oj-ji-ed zp<3aOz(#my8jV#?66Qq!dbd3jo`I5cpYv|pA3lqDcHO);wD<6BnxRx-$cL?1tP10k z_gVBE_BLPY$_u%{yf~;a^f4Y!!oWBK2)cDairylV{;*TcZpWS#x(Rj znqjNMnGyHiiRhCD(EIR+va6L^4$=y}2vLPQ(6wSo-?k5(@IkU<_c-WbwFn=x#W3)^^r& zIQnWEURrq<9SiPCh^xJi-gATUL^}aB_@VVM#mMi4Q~Wk*h_9t83;MhabDFoulP(s* zqb&0I;0s|74=cTkpdU0OuELm)ahiKs+!^MF zK1E)x>k1|k%z$$3_#VN~s1d-zU+3DQ#}31dwVpK+KU9Vr?d!Vkb6$eXTb|%4g{xri zR_D2hhwI5k&jgxCATvB!M@B$iQ*e9ut!=S+3e)m7FU&wzY~HxGU~3Kwo(|V_cjG4f zQ#1y|pN-WJdT;2xTLCN5A|UQRruaNvn+8a`35(!=_%goR>?LZCwDg{J5>&Q30!4=M z^UK=&;;$>C04rc*u<`CYycO3N=Pe+W7-39$2x(EouXAKf`V9ylcdo*}gPYHL$zHvv z{ypdf=@f>{J)+IddG~x8X}rQ$;oN{549YV(7?pADJXUW1F0I`8S&gN|Cq3f2KOgV& z{Z|&&-*L+wsZ^2@1|?5Hh|ejbL9?ho>-O9BQutgnRqk_;@5t6y8i)I?bARo--0av2(>Tow zHv^)!y%bEwcEo6S3zwon5L|$OYfZam8W*C!B!KLpmtZ(@-De{sTbn0+Egwf3(=ZMG zJFtdlA}W24*Ec>qa7@>S2e8qhZD70eirbRZ-j*4!0~z)b_CNIkU;EFedoj_(ZbKKc zhhEatle2J79X|BCr)Tbln}vB93@RkszN{~QR}Z|BZ~;4B%labjqztO?_rgH)wcmVz z;Gy5KgWL;0kp}BCjllXhv$gZ&>(YnlgoMo!$fa;5+9eG7P0pL3+dou?5Bp+80-c zXAmD0i@3*v$Oi?}FLmsKyM_k@3c(gKQx4=kv>qY!kwyV+Pb2OF@L)-9^LOiUAhj{a zSK-4r&@EkMTCA&#Z`s`n3Zplrx9mQG8x>53vTH@B^<`)W{9PJ5>p?YE4!*~IZC*vg z#@@!sElpX;jpIW;Py4mKA8&XQUohX~y7nXG{k9_r=a1DV{BMjq+K*)y*4I8|$;nj1 zg=Zo-jFmio5-$bs36k_$_NMw4yjFimTVN=(U+bI6J3;5ou;pzYAnzkocE|yjjIQ+m zBD8iljqzY_!ISj0axFjTbOtRyvY(BYc=kE}v_ZT1xlinKK8<_Q0oUyXp*^_~9Ndt9 zZC`D$YD50D@19*%+gK2KF4UL%MElu}!javs+w-q|uco%Ds4aPV;m9XJIk_P85)?BP z6Xj8jCBByQt)Lf~(RWtFwXiG2fo+(DWhqH2IBj8of=SNKk3NCR?Abisx7&e&sc4_; zhkF__3qsEFTQG5G>2?&fIalKL@NmGE;bs@?xLVM6)t7RgaNUlgL;s?ZO|h;A-Wbs^ z68VylZz_=hB(ou~&-pIY2u*~E&0*ZZFmci`4Hpo`hU_to zqeZI>Y-sZ22l|%VHP6`$6mAsgwfRB+Mv=Hm`F|<@uj2nT{C_62G&19HzR%+O#r$8% zkn{N8%aA|w{oDNi0skN2|1?gW&i|TzD&NoH)T{Wvmj65W|C{_D(;3q4sBdNzz2>3OEM>NJiOhO+Q-e16Ltc>L;#{0~%eap>9Lxcrbeby-D8 z@`neChdt}7=q^6~*`kWQ=Y%pd=wWBLga}eUC~ypO1xLE>L)g;W2k*5w7sWhr$)WVO z(Yt2c4aQ6Zau{0=jP%AOz|L&T25*NlAN!OaYcTO-Tm1b;YA137b0_&`jd;{VgXzF(02Q0h|& zN8hF69tmejSRi4EgsUa2mas{}8zlU(gbzvhvV`wTsP5L~WJu_daE^qR->cJCO4usl zb_sta;d2t+`b&+!O~OqQmPxo&!W;?5OZc(me^Wvu?`GknR>Drf-!Ac067G`Fm-J|oBNVrf&LRpR3%te4Q#|C8m8W~y1oN*x-w%1{i?kkBJxj)YOXA>Ney zMVHejp~;?SZUzWHzDB7%#}1Il;O0d~OC4%j##-TXz7C5@OHau!DLQ3kReeKEU9Bf~ z)|}I4<$AK`*G2c_&xUR|*C|_6AfZ2L&UC=}6 zrgp&dFUv13K7Wb-f)#~+-vwtE`b+Z9E?hxK_|;c^xv{DtSXHT`s(e#<&4%(d8>$fZ z;G+C13zx4bI{!SzsNc9@{`^2)-3EV6WmRpkX6+T(Gd&gMwI>8UYpOhr4ONw%wRQEL zhKh2g;@3<9rz%+Csjq5i+z@P-1xiF>+QQ1JwdH^;tJ+wCI28~_8gUE%D+)_B|0{|v zF7%iBmKUz@onKr)xnjTkhK8#8AZfsi2G3g51q@|FOy#Jer>52utghme#@Y(PI@z-U z&LkzQ1?qgyjLMl>as^dEow>5Ap`yMfK*&gLu*lVOVR?ORP3<}>SZ%0TQxgoL6(E^e zl^eKDXeU)sR~xKH6@%6FRpk)&#`600#C5|JwG|Yum7B|Jf@))3Wff-xyrQlVu5i6t zTVKAB+N2>^6RfCKs=BHi(@=`nH&z52>nV+X4UK^S#N0sl<@NRDSKtox8o5CMJ9k-$ zf9d(l{iP+#&tF)@*%{9Sle?t|Exk2(tdSh(@45DJ_qgD{Czq-7(azmB1c%hRho!O9dk(wo^<~trv zITFGB!7Bn)m0aDUq+81iltc3^{F9<{b4h+q_cv73fi9y7T;o|2G(TEPAibc!7VQpJ z62iek9K2>-ZJlUB6wU;QlC?gdAB9`%L2wI^sC=qx8iIA}pn_3}YWjBgg~Cs~ob&yg za&@f@zwzl8_Df$=Q(Nh$E>qPM^lwC0tSk?f`-63W>FyxqRMnEH@|OpL<po?F@(RS^`a;iczxA|s`3W(3kyHe+x>yY25bF|74Qe@*h(0=N8x0C{PpFv>x|uyatyr+ z6jg6U(CKWM{0%k1D*u|Qbv3nyt_5CI%NlOQ50*EqPlV6#1o$x5>-{1HEzS6HI9*G_ zurKn~eAHJ})YVt|d3Y=*!wn) zpI-}C)3%lyldI`Rx;`=a#lz7I^;|zcVOaaIr9HTv^5g0sD<{7=k$hw2#pkoY6XuKO z$7*jhKelwH(Xal{2S47evckFC1aPaKP#*eO^I7#IQF{5hb@f&2FtV}y`sc5#X<+jb z?OZnc(R6e&T`rBv{p;y*x}O6toRtz z$$_N$OJ|POkNL2qo7_geoL)nYNp1NCbkz!Y92O$+bnJLc%)*~^$Bk7R>*}xYudA=y z49CH6^jUa@vp75&H!%cg--E>$UvDxG8Z7V{j5h4!#Nu(*+|+v=2nTC6!qAs*)P_}; z3pFBhVVI8AU*LwxD4e}HjV@!Krla9NkB*2|cthiwjWt1?q~Y8%AhAZV@`le?PnBv) zy<%|xTC$39_?C7t<8uv$O+9w&@~m+9s2G#Y$R3|wS(P{+!4u`9;RsG{lpjnXHeyh( zs*DZ(<&~CkC2_jHZc|nLhVlS8S(=}y>ma#JeWP%U*nXr%?VNEn1a9mt1{-Z?fy-`4 zV?DeB>`_ZO)^w_cfmdw;RsuUR#*UM{)(a6d_l~xM-Tqr};`PVSk@3KS7pG5F`)?T^Vqy}$;_z68 zBG-D-J08g6)y~M5)5S$$jnMh19|kE+@X7-NgafS6W!?FD_b)1>Ez_FD+Vn z(dl!EpoT|*{J7#`iNLOc<@r|3-}`O)10#66a{ zDfcD|9+kVphDTXeq&x}<=ymAT2>ZU`P<`t(eUml{@Jw7KS2$lECDJM- z4Iur~s3!JBLysYl+GX3{kc6@|8eT1-!H*MEHG_x7H8?gLvQ z@80rOcN(#DcZC8fpK?@?ad&!Q!<>6P5RUV<%k z2y7#F`DVQR<%!~Lteu{dqjmuH?9TBcYRVup-X8GaklxopG=d#@iJSW4x|7SSjOm z$HN$}2QUu=H>;lNrS-m->dEoxS6+E#$zI$Rv)aGUX;)D4#dGOW_rC=SgFPf z$H6pSA7Fi7LOD&k+}=AJ>XT%|t91O2cRJJ#lq>S)m5Qr==HUJ(FkT<+0L;=ZYzJw7 z`AKIq@97|eU3c`^8Gvc>3VU{{X+)BmwMMV|tG;3C6TC_Sm0~%ljAUJYeM5c4^c>zo z+5uP@(bX_nJz?rc{d1%xtF*Z(Ds4_mz~M?!uFB!cg}7_Z$be&HvKm>Lsz%Nk-aE~i zq@2OwD(4tHH-l@`H4eov;8OeoTv-G|Y4*4^k0aTkk{4sI%@$6aNPMKJwB;#*BNsSS zHt0lgveHymBh}Q(G&Qw!gqk`xRZX2UsV}Q{e3ygjI2TV4 z9StWvMWv&x^f@E?hL@$3unZ24P#q^KbvG_gv&bcw%JG*0%OM!*L+RGIHBYpznW-uh zJZFOEOz@mJ2Xz^rrpDuep7C=>s_}Ei_GR>p?iv*sS*FWQQ(Y%3b;l`6-6Z8$;|@F{ zl`ySGdSF!9$dWXq_0v7OE*?!! zkA`c_XT2Yrs>Y%YW9N+S8`YcEliG#0oSCdLOz9j%9c;sF^9wDNjaY>FV_E&(0}v;==mh*`K0C0f2$1memM1gtxLfaRd$wA zx8izRc(ukIR^1OE59NpMJ4;hlFY+w*Dm5AY73q3>9P$I6`V73Nt{K}f!l6c3ZH2v^ zSEi_ym9Sl?8`-Zh$EB&`DqU(y>1f!Abd^@0(wE$;b+KD?u@kyTzXBfzisb>DNBK=3 z2^1;yLtK4Q7i-)BotX=nPCy@+fc`jP&bYoYy{?}0F4#+Vs&WT$fu0-v$~|XHZ?sPR z_s^XE?IP*cdjW9)z~4U)S9dGgM{GI8SK#U)7{n1Dhi$Os zOIW{w?5vedwzUk-`Im8>EV$NqzjCC_CSEgI zUGsc~+WhDkb$R7jwWf5E8toXThW9yphXs;yXe$IG#@n|0~dTKBG zXu4w6Y0j4lKH;N~e`4a`hvHfeuwff87UKH3@M4WSG82ri>_?<0_9HStt{sxc8b7LkWR*F!%h#Y|qF1_(DXOFq&lBPLglQ*XY_h{J-?ZS2$xMzL|FfhI z_M#6i-X#6c=zs|mwTG3r_fysT2gxtBwEF-y2P2Gn3O0i*eECVIaE@K z)RD$dsw;(57HG%p|8V8G8sh`5KFsZ@9j);}%hdEyQ%@sf(%I5jU0Bv=`##8jX$$6U zScjOVAnSfG@MFPo9Fwe$F@4lp#^P8zn)Xbj^Y*stlkqX*LXsYe|GKuX?;)gLRR2u9 z#wDw9D1Y1>S9Gj#j7(7@mk;lC29nE$m57XCL$t0!zuS~b-$pOnNF9;i3XAe9^)Wi$ zg*-3gYUO;GE?MG+FN2#Cf**pbGg?Nniig1;x8OKNI@QRpiM54(Ok;f*j(I?;>H^;1 zaP=`Q_%ImPRX;fRvEUhgv|f$m`Hbc*O$FMO`VKDN^;)m2aYvd{rL9TnO%4nzBRw@T z%eVplIIbqiV~t14I+{JT>xR@Xrtc}L_gm=WxO$}wYuuC(mG4d9A6G`Zu0xI`Zg5R# z@|4^Ry~b6*d7#^Kla&>w=YAvN=6)01x__Fv;IuKxYD^HmdF2ST;})f!z_nF)u*OZ< ziF97u+@ttic3F*1Qlsajl}(!j-x>ZZ`F80TYn;&AzVG50aa^Nr&~mWGqdXXTG}nMn zKX&d2ns4;d+%2l>fn*iHc*bL`!5=6*s!tCahPkt2e2N-hk2zv5%}Yj>Oq-aZCO(QW zEI3@H&K;vVexTHTTwTmJ+5%}-*nsn9V=prkjGzAjzIyLaYRZqXo_eF$2JmHt4agVk z{s{O4u&K-86C~nqAbDEhYp_N)F-Z-F??pP-44cP#=@_Hnhm{eGw!o&PCP9uFs^k}V z*7q*7e}`^=YkcDPVQToFDVJhe)s&Cd7DlTQ&R^0nA9CRpWYflr)}tWxjQ7LDY$isZoZ8>f1H ziTTE_Fcy46%g-7=TE1T7S^aB_1(MGi*EZMUQ=y-h-H-V!t_KC*8c&4Z2i%i)Vg4)l z)_6Z(nCgx8vBM`$O;#B=wZxf|TtfBULG?~fQpv$ds%tmwEUtoYqTZ>H&%p^Q`O%3C z`zzq97!DnKK1)%2*#k;#AQ;y7GTtArQY$?S3jy9raL^_kYdkI=(y6>CZN#lS9F5nK z-(uYuSD%enf^Ei?{4J`3=9S`q!j(lZ3$Fw_<{^x^1V`EScw9cD$I(XI$^+Zg1oCG2 z2ZzCrll42Y3D!9KDyi(NOiNEz=@`eIr76n$80NpYo)_M%amN68`X0yg{`;^$bd#=+ zHE!1f)~}%OMC0yY?xC|ZHph)sb{jW7S&d&lHZZ0vL)L*T_HTkiP52sglGh^Auoi)F z3%1d{Jhm1w@l?$39BpcRz|Hf^9Jn_2@isb!5#JcMcx56QjB&t+tw3>V56tupUL8U6Ak=z|0D zYonEjej=FR`)Kk!S{l2+yQ{-q1?d(6GX+_laa;@|e(yZ@nLEy1m{ep9!rr4KDL#)42g*W1(+jpl{HX z=ZwDDUm@8VTtG^;!A9Yy>3RXak3ml*>*bixS`8+ zvGpx$8tJ9mtm7XX zrJT=?RP&B=sQYpKobr>5-%sZF6M%IIACzv5Tk|-^!Dq(0#kjd6$an5b?=`xTsydH% zsDrpheP7GL8h4CJQKROh_L>@T+2Fl%2J+x4;5@oMp9v$`zpv}_^DjBni6=PJRVF{+ z@i46UTW}l`)6_&fH!u-(n20(|oWuLhUCa;K#FOn%6}X;~I>g6GH;#_u7?Y~TU_2c& zC%tcE&xo#Ad#5RNqC+jfb@C3957Dy0Sm#HozXGS?B!?4cx>{NNqe9`wBEv9i~zj zC-<=}BHkfOwtWj7>SbKXKLk%SkA_audMv^2T!iP%2#z_sJziqTXXjJvS&H)JJJfDm z)k4D>2T29dC`T%u^ID8>j6;oCgV4yCo)e%r%bEB60}fhSspqvPz1K4*DQ67Ni81zA z_UCE82+~wesi8fEj=IK@jw|AM)$^S`Rr}Q@p|m*WF}YxHhG5{4%ch0 zQQqB7tXDjs+>Tu;rNq+4mOf1ozDd%Mf}AeuLfv>bef=SJ>a_ z+=}%sgk9LDqP-^1jkqTq?019YS0q%70aS5(laJ_)*{R!6*T%+%Fs4w9<3)dyMg1B07eG}L1xbDIA1TIJg`aVpd zN2?TtJpqNa9W_#oQt4{6a^acLF={NHQq5FuHC|0nS$J}Fl8QamN=G1^#fLhSZ@`=7 z^hOuIb?C)G5?=2``6Y$D zve=3_r=)Q41t^({Eveg3Q*i|fx8?S4nr#T2m4D$}!7i+?ud63c6OsTQ{pu7YD=%Ug zqW<#+Mx{48kAg?ljgmFTaTxr5RO7G*)d*4A8yjk_j4^WV(Mb=!p`!HpO%}s=VuASm zbP|YW8}Bl##oHcHg(B|o0>FR>!cig=l5iq8*Emsvp%p#g#Bjls`aw0rsvkT78GaR) z-Xg82+qe-2Usy9m)99tNSOUGMR96v;$Maw>Th}SdN}#TxrU~y?0u3EAM#QkD6g`h@ ze&^hc`!dXBejbB80$5f#{3<-7Y<}DB!TlP*EwDQQ+X}oc3+ySta)8%oq4y48Wx(sT zz%uW}v(A7iOWxA~3lJ>9@Bf|0rRDPf&g0T@_)nk5m6WowrastMzF{d@j)+rY`P>oJ z!_F0Wxuc@Gw60Xn`FY7%x}jl3Rd6N0EqO-L*;Twg$#wg?R(McUUpKY}gOJ zWmpm3o~nTbxUyejDV0!IyQ!wWu685dWn5WaUqiA;mp5qM zFwGcNjKhD5Yu414*I$A3rzyRpzOEv_vJz&#L8;>%1vtng*sr{E3zwf)Se!dYpPDop z#&=0m(^5M2NLR2v7kL3~2;vp&s``yJwe+rUD&hGX0#y|?Yilax#GrbmW*AHy<%3tm z*49N|fZTwyY?NB0VEOPmno=v5EWhBq(xPRBv*+fR6Px@v)1W@+uc=$Zr#Yx;%3oDq zUyEn>n$_AsW3WMei%wo@2-L!xQ|nNhT0E~-POu=IV^yna8X5xS6;-NKm9MG8+uWPg z+O-?_kd|SZvl=?gX5&WK`%P;DIPEC779JJX&XUp~DIG+)wjywazd;o#|K^H@O1$!) zg;SxRgz6idx}mBTzLyw%b-Lmg<492IjM%-uN}Z+fwsSRJliPqqKlx`_iq6_uTeA+w zNa0u@xHg;p<@M`OqgN$mk_?2O>95dt>r_k=@Gjsf8yim9TvK}r zD4YT{si`|Q7|BK9JXUcP&OJCNm$WrSdcB=$qXI^gO1a% zCaZrYFAr{5QXZg#y_V=>kkr2kR$N!NzA>;=P8dOFufGEO>Q1DHA-bRz0)S0<(|JLy zK0d3mu&JVo&o5ibO^#XVc@ld zJUN_;5J>*7#;W=&N~-Glqy`)iutW}S%I!}bCNOOn%3N^;G-YEc8YRC0tX9#T+Gep( zD~i!|eRL+k4wPL`wWe|1x~h6p=DS3B1s$Ex7$`!Azfj#w5UY*6P^mMi-=iZXa2-Au z=;_Nm&#iqv5;0$BFmpv3eQ2HFR8Krvf?I!mFY9w0cSzXxo{l#U)9*c&{Yl3?hb4Sp zhdH0>P`x4WY>n?pk@qOW8lI7+<30b@@s2)$J2gBfNryY$((o=RNBvFkKGLDrt?>e* zb%?dVW~xiCl+$@o<9RZLp46jP=mh>Hbl%lrkMLPGQsaA0(_x>|Ve<$bcD}Ch`(D#w ziNt$yb-MQz4c{T@&HvQ#UXfow%I`DfN`0_~&-`P}oWsBYov%aW>;0>at5+p-X}k`B zcZvLVjFtSte~HMU*CB9`XYV_jPP4Q_$=`LnPxx(?{DHYTUyk8l3wes{qZ`UA@n={Ebxd9cYL739LZPmM;%w6=rHiH4!s}ha7VACpQ!0} zi2OamSBdDOSLkD%U&~kc=n%SnqQ51=C-#mQPf0)Mko;YTba_~B<@7$oPlk?rg`ZC8 zCpkvGqDLL3oMSaU_Mn(fr(1{3qDNiAf3NV@F-oTgr2V_3T{@-Ru;$Ek4y+Y=dv$2& zn-J~+|M~>aCwQj5CNy}d>olG2*L9e2One9jgn!wBmo0b!!84)3D-pb1do&%R_a-#u z1_ZCOSL5{vJrf$d4#DdZdZz!FP>F%tDR?`jzSz%Vd704QbqQXE)C2oFjAufF*DH7) zp=b2Uga%Jl>-=3t&eFc7orh`hsSLqul6n|>VnT!G5xkwkpGWk-gkBV{eV|Olb5kqejDz zh+IrRG@;Rh9D^tH%y?x&gI6MW)k4qM855dvn*^^(^q^PjYeF;bcL-kJpEaL5UKD#N z@-=$3L-2M;f9td1VL6(Ay9BRW^u>%*lB^6~kKpAkvF;^gzZnJKj#g3rqX<4uID!cs-)`(Q(xflyZ&yJR+AaGyV#mphe}^ zE$x;e%>d`Cmi~2)?*laSa*ZR^c^vt+vk_}## z;B`qodW?T1hz2i5@Qw(cSNuCWJ)huh750$J;4* zebT;Bzeo_9CF~ZwJZaxP<3HK(!h#nN{*0e1Xl6K9sT;JM%LLE($9BAI!Rs;lB7UD8 z&ntKxqR(Xm;H?t8PN}c)BklA8g4ZniXXYn%ybi(35IGya(T=y%@F()yA^xQuuUqhX zr5;NBa64YFDOc)Y{B=8?+NkB6YUDfsJ&)iW5IL*A#M{$mA?y`A&l}Qy1K_O^ylSa$ z=Ky#C!D|*gC4RKM9vy<`HF6#RuT$`L895Jt*CTk_MSf$Q|1qQ&Y7Ca^8nt809UbEo!h+dV+e8i5o z-IObGHuDxcUZ>zS3x8(*X2-U$U4oY- zcspc$$d1<|cqPK0Sr4=0g#~Yyv`5K5;_a^S+f%=++fBW$!>$4FJc1XHay_yRWG~k% zcs)`NvtDG!D-paVX#>sNNX?SkhOdS>0oj@K!8U4m!UckFoGf)^G% zv%X`;>lM5sf7bHrm31IHo~qaB*`j}CM-sP3mf-b@oHJxS&d{TG^68f&c-y7izyNqY z!Rr+H^$dVlCU`r9o=4V^bZPhyy4(Z|er!^S9fFrF{F!x7 zJKiqAJ0Rtn^*1|SkKpAzH=D z62aRk?Q7PZ4W3#5t`@u|!HceE3!=em7Q7v%zOtTYr?*}3@&wPUFWT{T3SO4rnRQA# z-Y&sgWy+QHNIPDS;O!E*^bLR)7QF33uS99N8+yE1K#`#IvtAwU7yvIz@Jd9l%({7$ zo{WPzg6Df()AKrPTA}yqvZ--K=8Uu`zG1Sl?L7_c&Tsb z^yUHZb_m{9Dc7ubN6Qs@y995Q;F)!FJ6?~$6aLJ8iybd4c-!}A{>;9D9Zxmt^i+|b zH#u>AJ%X1bcwII;al=%e;O!FrGGxEOlxy%x1n%82~SJlctj@{nG4X82${sY{3f%J+lvE$MXqZx6m{D3Q;`iZ`Fe575U|ih_A2l zU$zR~Dv_Vr7qipbA$U8CzN96lw@dJLioWa^0I$dJC*_)bI(xZcQ?As*Gb(YpYO{tP z5V@HB6+1nT;BA-o^`z}^t!SkC~__nygreO*}pVsM$XNG7nb@eS7Le{ zf|n)ss&fFmPQh!E`g$`G(>ow|-O?Ui1K_D9oxfA?ykip6%ND%t(jL77;CTg4i9VZs zOuJml1aGU*Q{xhs+a!2lX^-Xs@U{wGm(c66;o(cp{H*m3!OM{LF#F0;{=E9G)6f&S zbY|N46TB|NpXh|&X;o0$)qeJSr@kprIE z*3YcX_pB%>TvD{OXi0u4zBcKRuTGAboijJhf@HuW@MX{3Svj+Etci4L{PLywOA2GT z=giKFL!qa{=}VXec=Z455ebv>SF9*pUWzmbzxTQLd2FZb<4=`_>6Z8+PvVx^S`pvg zW}k4nOc;vQW_A4$r1H1>} ziFh&S6Wm+Ru(qCWP3K_g>sn_#0d$xbK{d_d0Qv;l5;wQV-**#=SaL zDG#my?!5@xJUHhK_ZUK5;Y`Q7l3&6*uDH)Y_#3@9+0{{C0QbFR7O%vs*&I_F&HI`7feKt5M67rs#wu?>!gJ8NN{ z!NqW|(HKvNEh3p~S~3qc$w$3#4Du174)Ft!Bbaz2f7MH!v4| z$%LZ#;2QWeQ;HITIqJGXv=!V6SGPcXg4yt7 zC>0!UiQHaRhzD>n++{j)aDgRrDU%$_PoWN!6W5!8?H+~o05`LtC>EFvpEV2PJi+2Q z_+Acq{to5v^N=IB6@JMUkH>>+;4kg4o=G{J{4jDbnEcjAB@_k5Aa2SRiU-HT^Pp64 zG5i7~0Jp+>&BOeI72u~MLLK7yP&=4-DI^<>>A>wB zv0cDU@K+EGZiQPcK#oN)2YwN<2iL$I7h)MnefX3`=vo4c;rAEgxkAElt0jmHFc)42 zwUTmpR}N$1`b#knF;bmw3myWN%vDTs7@vXkQBM2HEOeVB+PFBUmywFv%ILz6$eVt7CacL1c$?q zK-J(HxKa$-bTADcxCzS$X2XLa738NS&lC$G7MMJTOr9&&1QXAN^ufjOBXL+pQU~6Z ziXID1T&NV~#IvDlF!3@- z43^BBOY-b;w^EoW>cbVcVY$G>$3QfgJTvVMu}K|xS1w)R7a)6-6PMqP`3Fnp%4MU> z4~j)O@k3B5xB@a6jTS6%#%y<=c?_- z`3%a5>p;{ttP^-wzFgufAr+Jp_k*;-V)*bxY!|R(UR;thR|wglocKY=3EY~5`1u37 zC|EN0Ey;;X&bxUiCmstGgX7`d_TYRB%!Pk~s=%yd{JuQY01k&|LM>p06nrPt3g*Ch zPzRU~e*rPKV;;$ids2$kX;6nx2Mqm~^0b+sq@EKWHE-(jv53&J^;az!aiLcDTvY?!JJQM{c z-U0Ezln`wfR7}d@j9kncVR-jE?BifQd_X?73z!2x2{nOh;ASGsGbx8BLGMU?xOD-( zgOtO&^3oF5hgk7A*2CwJ*I>yUv?Ld;R1uCTC?`G;@&>cu+o2F}Jp2~46)c9w7Gs^0 za(Gv+S>l0EDawiOfT~FycvoIo;^t5*%84(4m^-ol;a4DKa1H!Bqz zqz~r8_d+IMKKwFd0hY`mOLEI@hMZ7N{2CMpmdqDR^2jDbJd_h}ghXJ;9I+&K>r z|IEcY7ku{|<{9P08=!WuWUf_`gLO+4Vg`9wiEo9p!SV2CkO{aIUVj1mDOlkmj)#{J z=inN6SvAi0NFBKB6|_0v3iza}SQc7>x1L6vjqVrC{O}P&Jr%&wJRW zVDwE>i=Y-T#xqm-P%9W?mZ>jLI~e1VDL05p!m_|CAq6n{w<-MxSSMigSyQ_qZ7{|O zQ_mqIFvbE?lf+mLqz-&HWKHV8pF{Sf4&0&<@eIb;TuK0WfHD5IGY5Ad7=7o|A}Ab; zepo6WiY0a6U!Zs}#=}xGAs!fGUa1s_4@SQ=^$rq}I!&kp9VB(&0;n8Z4F3XEkvfkt zUr-GgeXLYE)IjRM+n^>==Lx0*y#V7mD=Gzg2S(pF^&IK|WBevHu^G$o2bLWk0m*|g z4wGtwl)+RB`iLMF7~?6aT1Xo#hTA{Iz5_|5`UI7N@th*%2~~hG z#*eCps=;{fkQ(s{=Urg*jZ@bkF)4>ne~tA3#+W-w08u%@;kIUV2mxJ zq97g^3 zLdxNvpeQiLCQ)$^4~+3g)Llpf#uy`N%vUTk7~_bj^-wt&V}+mh3}?q^V!APyK~O(=C~Nc zPz4zGs;H|_6}SdIwmZHLjAu2e^^h1G4nG66kUH?evRG~~#xYP{P&*i76{uqnm4fNQ zhxb5UG%)VHQL7+jFvb;7J>-zj42)-nsXY)4#(g7dNl)Zp1Y-;Ur7Dm00LFbaDhl!j z<6b_c*9+$qVBB}4zC!U}+>@s~Apw{R{{$72@;=Bh1yzD^pNN{EAVt-IagUrj4!r>5 zeiUWe7j=`yn|Wr zT*w2A`(D(c#Ac_6_YcoK)37kA!6RVx7Xfa-b4_3#p@=ILTLP1eVNON^+idaC* zl`E9^Rw$m-fxm);VB$kGrKobSWbRLr6I5CY@qu#Us!%(a_yjBE0TdE8i z0><;PR8Ir!Q(!!^O0ghqQXlRG>67~KaL5Eq{1jvXZiV-rfE>eM1^5-n9xRzlljPVO zHxZvhIq_U*H<)-4Bmft~rH!$#fjRJ(Pz6{rhbGCbIS;BqIq_Ri6Icu<`7hs*&%(R% zXG-SCB)KyenV}8L{NZwJ#1ELb55xiU;d@Or2m+yI|tiQ^ZT1Ls3df*)RH zh4)cj4xeEyMX^wB0}q>yml?gZZru~AMu7cv5i;I)t=nD|r33H$$kuc2zx(c~ciB2-E0!;@{X9H^5DzXfr@O>jdyoL7Kt z;89RKn0N)m0~7Cn1Yq^K_%3_IAej7?Og{&F7gz-z09AtH;R#R$nEc+I?0jtJI4lF4 z;pCs+{&23w#I+O3$nSWIrT=_0T$xF2kk~Y0X??uuK+UC=;ooVL9cVf=hnkJo z)A6q*nEV}QzQ$QM`CxPR&Vrvl;KvJ!iT}tvO8&9&y;7K7AmvYaBInf#OwE(>rra@Q zUurq!hsr}b>(F>k{l|a$luPF`>lVlwiX#x}_)uKR8THpzsVM3*HYzEwT*g2ZJQ%(;nqy?YU!X zIfKbMB7YXh2#eODa|$ak72nRK>e@c!)sL+ki0Qj@e%2dHP1dq5rbZb3Ow?#hgM(@K zp)3gR1^yNn|L3hnFHPz75 zr8QXYzOH_5NL^;CG0)yo$4G+}5Qr3k&fZA0W~#B)JwU^3axXa(=YRlr9~bYnEKI;R zz*Hm1-`6<6)dTrJ19W^mUH$z6{FVpmAV;0CbAXTTYCR1WvXXi(cMk}3ko-1Gip4Ss z^bZOMBw4q9PEGA!Ol`bc=jZUr03?A!y27EJdQw`^U>AtJ|yxjdYSV5lVt|Wb#sm5|=?*MlV*4SUuF&X=h zzr|$iZ}Vg__SeGU<0fOf)*D`sP2hy%Cb0~UjNpv$jM$9$jGBzPjNr`hOguB%c|5Sr zcgpw3N2LlJr_qSA`PO_6-;wXc_uzB+!TfN3EI*#l%(q+>X(pAz~>6+GdLNJ8DyThzvh~k!Osw86lauX zRAf|jE`vCuC8IT?J%h@W%~Z%#$z)|}X408Pne0sKOird_rc4YECNGnp zDatRP;% z7YGH#f>J@1phi$95DQub?E)%GHcKUom8F?QXR))avp89fSsq#3tl+HhtoSTm7C%dv zRhm_iRh3nfCC+NeYRziTlFe4gR>@{%)7eJZ>}>07$84u;k8Eyscy?@dd^Rszm|dJ* znq84ylU)K6Py#C6Pv@!;pYf*igPM* zs&Z;_>T+6gT65ZSD4~K-MaU8w3E4txp_9-<$Q8y4UMg)KrVS2kB6 zS2LH+HOl4WI_5g%2Iq$7#^&;Kg}KGKRk<~}b-As%?YUH*N**guGmo8ToyW=Z$m8Y( z=f&so^7whBc@=q8dE&g5yw*I~e1&|Kd^+DKpPld68BO8&vH9`&!u;a=()^nIx_ohd zdp;$S6|qE`B3fiE;)onYTv4znT*MRcMM6=9s7h2LY7w=H+C>TlDg~?pqXKq;b%9fX zM*+7WwjjQMS5RC~T2N6?S0FBEDWD2v3l$1A3+Y0mLQbJ$p;KXSVR&I|A-_;qSX@|D zSW{S6*jm_LNEN9Rv5GW{*hSVwoFb1RZc%Vid=amRUsPIDQB+kVE@~-iEkf+q;CvRb zFUwcptMF;Q5ug2Y4+!s!eqm?i*K|gGduPP6I-}jXGt#-8QO@g(@QTjpZt09{h0drp z>WpZo&S;Ls{>S?n#dR4Zf`3G>W@qGbI-~YS#QuoZs?JFL5v3{u7GjhwuttP>2)KyP zcmWU5St_VNY>EXfh)mfm1;iztWrV17%yL3ZhG)ehB86GSh{u|&Iz(f87L_fFIMhTW zT4!?*hurL7L?JJmj~J}Tu0jO1WVa&z6mnD$eMUKK#GX@*2O=*vCmwNEoKuRZtIH81 z<|v^oB2H6CBjPwhM?_q(FdPxb7YY$^Rl*uXT&u7h5vP*NLd3Citr2k^xm-kCd@c_W zSDIUah!f|wAmU{66cBNAo)IF>G0zDR7oHc3h!f@&BjRfE>JV}5c~rh^J}X}n5oevx zLBw(MgY(1ldHH-qTt$8rBCaLB6%nT(QbEKSiP(rZCy@stE>;weh$|MABI4>qVniHO zAX}hNpjkj8;y4A4h`8W_a6}xxK!}K|DyTukwHCA^;#3M*h&XnkH6qTVkc)_mFXSQO zN((CxapJ-jM4W7q0wRtsGD5^T7C9l}!i!=Nal)cvL|jc#9U`us#G4q`Ixvba%V+U5 z5pmX?Z7%QsXe&$qZ1H*jcH1dNi%22Pj1Xmxoo!$&VoccC`qd%A+7V$YX!Y1xWaj_> z-~a1tAXW!+$Y4-(q)v!TkI^yKF&}$Lb!YG+b$n3d$&ixL>qGaF>ERS(E#)rFWH7o> zbVRI-+!z`B+C=08h7^;(jt-{BOO*7XM@plkfR!;cdqtj-^P?k~^ZC*YDF#Eqx?|Ttr+K@#8RlHr9CvGXfASqa7fVVWGrwN1 zC*4CvdLC0oiD5NGZ!kTOypWYwv_j%|WaDMca`wepBuVVEG(f|0QKaG_ze$f|b4&M}i4Hayhh+xm&3c0RJ(eCT1){hCd#>T#RD zzgFBeX~VI@n~xvw7N(Kn<$fbe-gAHX^ohxBKE8QP{EZrSl6#HeJe|3yq}0t&5IVy& zcU8V~NkEB7Px;q($IpL!LVv=eKL#)PI!w_tQhN21FJ{ZH@7DAD>{FsII<+piA%XM8 z?R=gNGylxWXX3FrISszITfW~Jp3gp6_NrIfx%#v{ug~2dmMg!yM1z%n>*&|;W~uA1 zpGRsXa<_dy*6-@KYkdqebmxqV547~MGEiF6ZTuWp-BO*#lkPc8FO|AgpQ~lx_{8l| z*k5pWvUXFRO7XewQb8;3`#g1Z9cmcn8FwOY+GO8_%XXwMy0n*$p6#(gChrsHa$Ijp?Aa(_O1Q{v^o~2s*HIy?fvWHH&@KjT-xnbyv@4i zbMI<8K3|?Z&QghGxS9FvNM3i#)Ly%cdo17MxL7N*Og@`7GO4jNGzdvO`s&QhPjah2 zhjtsxzL;R}^ya#2hr+1B-1xk+VN$6I`}=oP`>HhfxqQp!PqBwH{5=d`c5m6f{{#lT24k54N13d-KC}J!K9GIRHhYa85t&!qGe@dRz%37p`oXe zH-|AzXk*%lZ^+k=8Rrog$Tc22*45uz7uR^$6vw&dloV5w z9!0Bny{4sNRDMp+&ozMi+XN-|8?+zNky03>c{2U=mcV45MX9Idp?q~YdBNrtrN$8_ zr)}O~*Il@6*zDoa6EnS*>ab!?DwGzkoiz9I!7p_dCuHguyRN8Z_upg@<3IY=xPI>y zH=h2~xTG(8#fK>x{%1!#WEc&yVLiMn_3^UhoKYDe%PVXR563a=&v!U}@upsV<7Mk6 zrpwRO8s=9wFndngM^TSwOCry0DGJXy9${@4rFUoCi+)YnHm2Sa18RFlPh^Rs2YEE+ zSR3VDGaR|*)V6{B&&Jv;-!Rto3a`cUfyhL-WbcI`iwf)Bjt#t88v5b(#=U)$T#^f? zDcuQcu=?`gjNQH`GV`A9-TKDZ#kTx~dLOM?$MC48Y3mo(p~*diCU?xwCf7Z4O`*q$ z8I5Kyde)y9Kf&QItBkh)cU!eFv=+8AQ(8&wpS5aea_z<&cDAL4v>qOh`_-0CATPVx z(uES=I&ZF?65W?fTUK6fDt;~ijk&Lnj>*wzrh>P;B_oU_f{+IW# zmt;sTWM_LSwFB*`)MiGcR0MU=tlvnxfLo4VMPUo1$Es{&tWFXf-0!EnL>mL2&wE z`H&%}GYdw)HQguw@~nRS7xU#VqnEas?%&Mrm~zy)d09`v@Q&0w7K>+|@4mClnH~MO z?ZQQcil_3^XHBtIp!VM1I*nttm0u{j=x<2Hb(?y`Lhp~0@}7)|Tm0?xh7>HwmQnc{ zw6e%QV_llT)0_j=)%){q4r@;9xDZi!U1gF@t7l4D$BcNctYuJ$YrTO|6o_N>q7ORPmE+alg6mCJ!7C_^KToo8Keq}IThzP1~T%UCrbzPq!?1t40Srjuz6V@{#j^klvX-#ErdqV1FX*w3Bj$C&^* z5%_r?Fx4FUOJ@tXw(FbG0{YNiv_;oznjtaOK;8e&R0IE99<-yK%Y((~Y)2&~Be}c$ zXhtKb3D;ZiUh}{Ia^1q7e1j|J)~Shk2j!nvj>}ofTI}Gw%4g{v#>D$kTaHbf6C+n*=nWbG*i8|~iET<8*7U6R99@)@we!hi0&(L?G3XLb*I;`BnN zB#|Gim1bKy&}K(^#q4q^r2~iTj1m?xJSDb6MBvNSQ%s#u-1G zaTiq|c{{1JN%3p@vEqggk^9Rg`;VQR{x~TfKW^Nwh8(T?_;J5ZivKf*kAKyKr;OP< z_p^c7b|;ysD=V(P5BpFxcdkm+l^fon9gf`-2ds4;-?Q&f%F@z%YB0NsQJhfE5jxCP z`Cj$;z;*RK(Q5~`v>Qsh&g(7Z68=b~T6O#A0V_8+C#FXqi57f#_ioq9{_3Y)$6hP1 zvUR$5w$;TC>6Fdt#p!oKtu`Mv+M0WI*moz(kD=`5dzJ?l_$}&c!rjn>JN?##|Lr;F zpQogM+rpFUILT8|Me@3{374g1a7v16puca(|LIxj%pWbY1&y|ukG46+_`lgQlM6Il zpLMp)>$}=ww9S6B4{h1?8g28R)BX3_=C0{-C%FF4T4$^eneUN}Yg`R|*QfVmr6_%8 ze&YpMIG=GyTy<@yS#O+Ia#$-lFyaPMI%Q|4?>C%G}r@~ix% z`dBWRn6;~9gg^u_O>6dVeHos7cW~B%cg9MK z7aZDXvFu~BfqcSBoMs6|E_nA6)V(#py z=F^+^w6%`f;{JH~`qU`hUZ$fKnlR6oUmt(u#e0)=6PnB(yfV$5WM9tI7P0kCdtb3h zi_6JR;&J(+pwhsA3B!&31}A7;D%gB6Tl2*1*-tA@+}NhLW$BDjDe>>^R(OAT=8{w0 zy}nW_?B1OtK~uR8ODF87`H|+=Ed|eZmngoHeI9KbkuvGf*n29qS=+vDpRRSS=s?ny zf(ciajeN~}^I2(cN$suaPcQk>9FE(Dr>DH0*(!%Hn5vvZV~bR3o%f&Lao=Kx;n4>Z zy!uu*?L$)NFq@ZM4DeVQX%jbTC;^=kff33Z=i^rXgfm^@ncHKhGF*irHBA z`B}#1!#*2rj?YlmG@BUdA9`fwsDhPu+is`Tjj40zeo9O)@Q|M!Huk1weQ}vr_@??H zdXa-y(vgFgqYH@k_|KffGZg;r(m|rD34Kcge|6vVkk*rz=w@QlLnL1H)a#A&vOl|} z`qO5Aevqj*j!9GRL^`%3=4o3gn9taHu9RM|)JlD=nq>*6mn$z!Jkh>oi}+*mkN7?Nvgo?$LvyoHZr?m@S87GM zoz_bEFi1bV`0SFflb^j09UG{4*?met!KL|WGpFTBJ=ksj%B#oFxcFmLBl;(MmRZKQ zikE(p3+$LPA&)&E*|5m+db#E+4c)=+s|ESmRdUm(@3^rsDns=;_m1@i<4-O-23lDT z8M}A9ZM^QMo1uqSCY2=V?iP+PboHHO^=NI|gG08*1Mjt$7nO;=E3Z91T4jsh_sw!! zJtj@`u<8+3w$bB#DDPBk(vk6M;+dsy`s!U}*S8f7Jf)|+;eFAeb64G+b*>#-y5@`B zUV+VI$L~*fC3eV!Pbm=Ow9QX1@LG9vLs@0~n`zH_^-x@Nv?H}Ra8R${9fFz851FT9 zm{yY=+U(~!ty$T`?huDOn^W$!CxQ?A=9uM%To{Cpuj+|=1c|5Uh&w(jA~(yW;UX?>qhxT35Q_bz`zZCH3N z=i#&TNR``kq{@w7+gVy#N-t6c*P<#Xe{F5jd8ty2A`wov|EBZuqfF1A{ee9c(0dut z%#3KFBN`FrKPP)YkA%D)+5TOjNVY%A;G$03N$ZdC^uQ*E+k3>_z7%E8UoiE49XBcX zjiw^qgQ3i>5>&oFITGk)?`CA0&Dq=Q9R+hB(2|TgYE62&DZbE>6)4 zIe_kmLkUw>srP*JJhB$}`FjO;c>25R{o@`$n${&Bkyg_9dD_qV*#k&^c8WDO(AkGe zx&X-~06H~s0l<$`V&aU?1!sI0f1L64r36}&e23n<_5npRhSv9*xjUIcf{Nk z&Z2$At_Pf^J>M6W5O1iYdwFp|wOPgzQN^se)vC!>-=?dL+ZJ|$Bh&wIuexvIYkf3L ztoq(pRSdm0%XNs&rYDc??ONm$+j``Dd+*9kre^zd8e00l&>!+hb<+C&!M#EUTu9%T zpU|S{Y1XY~Ywe~i`_mEkoQz$P!j^2B{w^hL;+pu02jjoq8O|PlV3WCXN)Ktr?1j0M zgJ{UWFE5@YJt+AqqZB^9J~PK(ZCh+~P=)>i#?a?8-<~MbAL5rWue#rIlW#|Mr7m`B zIF}iIW!5s$_yspbCaI2=ifsoIu5%LBUT~Xr=8?H@!_q5l-eQj)^~up1;?~^q4WG*= z5Acv$EoZc&I5ESu{X^xk9!EuMM%|5V%K3RD+RIW~6Ilk+BbUW$f z`R#mknd3_vy>%JRQ?}WKA1!hIV_?R(6p1c{Sm&+jqbImUXgZ~*=+$ifs?E4E=l3oJ znTo%3DUe=&)sdvbRI-aymi~vm3bOy*UIoUFX$|Y_p)li{@+ZYi{CCb>N#oMhw*1+x zu&(P=6{oJgv^PDi>orbY|D5bUId%QJ$^Os#6&Mkd1u_1%!#}y_>YCbZZhV|lJyXeI zSJbuxy3bC$PBFNxo3Ki}IA+hqoPNoJZZ@&)_lkQv8}zxs^S703IpB7GrbU{?UWbvZ#^>ED3p%*nGvnCt1%kc{dJnxA^z=ph zy8)A>N1rrSjbK>p>K|)#f4;?NsY?0&r$>ysQ#Ysglew2yhDDeMY$+{~bJlsdbAr}a zdE8)F+WYMzHsdIxOh-z0{Ong)H=^*)T+gIFJF6xyes}(n*-Szrc9Si@VquamgS-8xrn`)l#8#(61hw-7K^qb4_mIN#6(2BKUa5qUfX!5e#ZCR>L1@^L5clh!fvZOK-i z^@-a`k4#=Re3bgFGj$JT^FQADgYs<2oO!$9>CiJ9wYRWVJl<0)f4aoBa!i&((ZHNB zYSTBmyuQnQ-1nBd`S861UVIy@^|eENX?4yq$Ap7dEa_MMp!IcbZ|#_9hXJ38&zP-BAEq9y;^hW>xp zvmo1fI|5zAax{7%e%|l0A?^M!<}@1p#(cEOF_ZqgZF1LzKe^o%($xy1Rp!#Zv{lz@ zw90=@_}^@myC(d<)1#nxdp)b5w}GR4T7>FI$FDXzD*}toyNnwCVV=F`xpd2z)r&lq z#ZEoIwplTBmTSUiYYkfdob!;UDOFYe6YZv+_VE{Gk57hr0%k`1C%v_1u;8 z)27QqJI;MNzBoN1C$F;PE^~9y(=(IAci&_#o^@b>g=~VC@s}|pdkosPxVB7f*@SiRH?Ks;9d4MG-EvP&I(!K|)9@#uwCtV-dN3>WgWmW3Y z44beSCfY~O?o_+RZqPHKn-&B+e3tP(bvPsL)v`734`1xAlX-GLj}Z_0dEMtU?Rh+U z^T!nPN#}MAbsqF^;dJ}S=L_^SJ{Z1N@M@cQd~wK%+yP{n2L*cgk+Ho$ z$8G%9MQtQ`ukk;22>Vw9H)8kJl}%f*SM^>?*{FzQd;5xz>D^MQ7xwE@zgxlLpgB8Y zjq;w{+NRaoRkCxJvT71Wj?J3yn7h~4CB|Z0PpiHD^=92vFbwD>@=+Mf5khU$wcwx`ctEZiLU z%=a?q+Py@&a@>~NRQNf@hjFQ2)+Rmod*iTlMCt_k>68n_1A8YN>7!#addE?ZnHC?u zxY$&`{kCnd^T#Vgw5dRjS5~rgbX-8L<>@@-yyTO$K~^b`8tR6g zsL(z+m`dHz+bhK|Q>QF%uqHD?_}%E}pwwwQo^D^Q`e|uc(52&XjeWPP&lp`jW$>bP z2mL&Dl(|%VT(a0oMdh)YW7)AsUOLUz=T4cld0$+b?YiTZOY^7;W8E{?uHQcE7~TEQ zqn$~Wg}v|lh!t#Q6TUTBuGrUW=#}+u?}sU9u++yE-R(M-^~bSH@wa2y+_fhc=8c(g z)jqtjE~R4cn*-9zFG(EB{`OG!*JIhgGjQW?4`iGl2eRq36+MkVm2VznCUGDO3c%1P zvdeYe>-@C?V*mSd;0AWTUwdC?@3=wN!3zg7En0&f+4UOt90&fKc;`LG-=_YTCIy3@ z@PyUNU#l{+qQuz0C`Ymgye3-x}^M9qW}?lHz6qwIGT{LyEc zeJb~x%lld75ouS(8}}HrctLZRkx}l!qO|gHJLb9^Ezx?oCw!rPsOib;!|(1$!xIxD z!UkWw@21us+Ll$`=GJJL(vdi2hR3Imub*^x&S~+hb$U_0t6-GYg=DD|ri1nD{?A4{ z>R#SsYqHhdd+*xzXldqnCa52GygTe`^X|j%ybeaFS?|33_QA~BZzG?NOWc2__L%Mb zn`Wvp=;Sx>q~V*zQAk z%%>#hb!uZL_mJlG=~n$kV?$7xa8=Ll+bZYmKd-!Ir)mE&v|{GYlV+^@S2L{wqvg6a zPS`sBMIDveqSfQ3=0owDqHbbthsFCGwwsq6huz zv_#)_uiyG5aRBQYw%DVaULOn;lN)d7{uAa;7n`%jkk~1dbpL z%O~%u4V)Qa9+P`$KOM1&seG=`z9~4SEyQ%5qV-nKGc(qGI(?7!qZbJl(Eq8UAVcB5 zTtY^aGnJ!FU3GRvMmmSo#2ZxZyq|Dt%ANCcL?#`vT@bMagH-=tIeTCz{IfHi&RYu% z$~N{(uf1<`UABj0CAtQD>*<@?vguZmc~aGRyGwDY+sEunJb&13!H}R0rs|^(?+^!A zN?*P);Pw-Jx#UurnWH`}j$U4I?6$~dsdie^?bL_^m1!9dLW5RJ+1b~pq(r`3=HPxS zBVtr_O+{7rWv}{iuIh~yT`jS9yx#jmU#*QaW3x=&u^tN=j=H#w9D2B{?Mvcl|%4V$al7<|h3=9S#|-`sP;gCyk-4M=wwRdU$!|peq%p|M=rhbjx(^ z-Reo4+lri%qE+#Br&B{tpEo(L<-Kod<$d-njlC}{dKXXrxLGl)XleU)tCu;O?YwV% zJh9icbWT5~gE2O5106$KH;HARG2Q#POxkv5&>ti3^G?4k5Ttxc6;}3zW91S&pD9uJtZ((Z3wk%jBnhTMGIH3 zXC^;2tsUL1YGBa?`{PThY|K@i6+`^OR=HaZ<*9lKc?@yA)!{dvm-Dv`iGDn|NFdX; zl6fU%@TRHH`P=tTS*~=gb(3Z{{=1}$J_k_S942yIVgXRVb6xz&7R3S XzZYlej6Z#-*SfL36U>Ea3Znl7h7@1- literal 0 HcmV?d00001 diff --git a/sdk/binaries/tbbmalloc_proxy.dll b/sdk/binaries/tbbmalloc_proxy.dll new file mode 100644 index 0000000000000000000000000000000000000000..a6a6376b698e0cd4c3080de7f0ae920c750d228b GIT binary patch literal 85896 zcmeEv3qVxG_y1kk1ypcRS;hCNSeU5`_yF;-D_|F03&mGv0m7mrua;dOX^JB1dfT#x zW~GK^S!O=64DBI^DflS0GG7^1-(19RnVI=)|DQAW-rWT){eFM-|2@9DI&)^uoH=vm z%*?rS&z*Z2Hf}NJ%W<3%!Rh3MVht=`-j|?W z`zC|6iTxOUL{|DFYE$lEQUb?i%x}Pr^4paqODDD2~Ij*dGzDT%9 z>MIw3@(XSnO6iGTmPj{C0|ffQH3z(k;PKNFoh7?~Pz`8u=Yx*=5De(?(^I=TU3%t3 zj%!Utg@`bgT*ysy`tIfeb2^IHSS&_4$=Cc=bf)Y)5OyOz7$eqhyPN0+q4a;mZ$JHN zn)m0pnzSB>KI9R_O+qvmBUWR_I;{!67g0e!M0?S*nq&7Nnl~Devo)d-k0I*S2hn0o z#G2$BM7<*sr4iHh35fboY7nLD?GTM6R+FjP0O+BnY78PJr3MnoSHYPLlo(o<_YU@mHMiB+$A5WP#d8S#jwP`eRC z+>cuOhPYhNBl@Qa(RnH;%0=`eb)kPIqVKyR>OrjTqUcv5E@_2Gpw#Qs-v|kX)D#RP^2D(=qc*&W7O60)Z`0P_Z}ktdN87M zG-SRc{UgLBkxDaQCu)A9#(y9NR+>3wB;O3;GKKm(oQ5Kd8h@2Udxz#O^Z6cX z(n<~GlLQ}-b}!Lnm_w3=KZ>YDF`{Rw`vw|3M--y(s7D)!Rn0g=7pZO!%5JCmvWf<8 zQ(r_6QKzR+sUaEB%$A7Wr`8@Pk~G3yZilE75uc*Ib)oLROI>(nIHGYhvqzCEh18>$ zX{Ge0_4Xj?uP1fzt1*Z!bU|b%sisjIop9N$SxI_Up&@3LypdM}UJZCP@IOfd`eeT7 zTq-Z98r5(dAGM3O#fK&F_O@Tta$MA5Uf5?S4G!fv$N9+|w~h9@yj@kv$}-jMaRKIS zN#_wR+k$z!K9n!2QCQFMc4L^HOYi!S?E+6lhJWmdoQ z>djG>j)qbn6bnb-JaasG#_6m*!3*Qz=n^bpW8vk}rBv|3=e+QlLkqIosSu5ZJNkg3 zM94#)0Foz|x6KR-HH-#<)*zhcg{wy4=MPA*gyJ$wOGLAR4Ulb?A7YhNDJW|FqRMK0 z1VyZmwO`S-;Mq-uisBQq0-TkLR0_Uu2WuU3A!wp77-JI37oV`Sm1-OfqZ$>cF#t6{?syP{ zQsD03o#d=^8SBD77gYiO$B0fgb@gm2mpPq*@ zUg%Zx3jh{ZQ)5i88%SFl@Pg)hBx;irg}_V5e8niLEd4Uw^J`BT)`*jLmKF9>aMs@v z1>>avK6WRszV9IK+|3tPSx@q|UUsT!(<}ur+IqbL5OqSh#{ahScT}EY;O&+$o#?MG za<(bB2a+lWV%*xjMUs8$Fq}pcf@%zLaqi&{IryU8K78zD^FhcRL0Dfn=$$d@qIb~W zh2#6_s3Yw$vmyCzTFW}eSEs-ORxsCDfjJSo+x$IW{7*}3b#MZgfK^8{?7@J#3r(qv zX3*EX?=9_(_Mnn=IF=I*B^m9i^+@A^!P@|ik(f=WiRF?{PYDKmg9hg8Due`v6vHEi z@hOiy{7{9f5yK`d!Mcq&n{u2zPEq@zqgA!jxgUI=xd6UC=xtH|VkDjF;&f;T1=Em& z=1f%H2*fo*fxl_C{QpO|*Dss(F!Q`{4D%g>_u`vSgmB)GI07oeGHJ|r-NV1gciqV! zx~%Bw>tUv0q5o(FxGrs}7=QoIz)5}vB+L?&*ULVoKr)L6N|23Bu~^=G)6K8XUD{m{0OE0 z-T)TW=ZgV3-AWq3tUDXPRYZ@`>&pu-xJS=61f%D}+n@&5=pk+AwY+UeK&?6vDN+q( z8j|&>H?)_J*B{!ydPx72pXc?3?I%^w?sE-#_%PuzE=X9yMq#J1Yk49@K6Wp=#{K+( z3Mx%h4=$6}$R*^56<4X}y@Le5rvirFnYoyERu~HVwdbst_(Qe4b01%HO;tcfcE)Qk zvcp6p`vTIid8Gi3pV(ACg6wI`KF9H+PNxyu$>K8gyywAqyOy}y{EhTXcMmMLp7KIv zynPTJ@%pt?a2TOnUQp9UKse&qISfq*wE8Eo= zZ1L?BwZ1?bx}MwZS5x#))pqKeEmy(E9-q=$*h?EMd4J{Hxmy{rSlG+Q?lb>VN_4e8 z;x!?ht_k-X!)Rhp?Quc)Q}zFs3vV93!&Oj-xM;(fdB(o#&@kaNH2U;@+8j7OpfjS5 z!mr$#y-G^7MGq_fUOn$!N{hp%ZjZ}vMPa|LxX`w;sR2EyVG|RXj`@K!uS8kQcr*bj6_rGEM z=&lL7U{&)<_t-`K(sY#KM;11+iWH%{l zeIjA~jJTV^I`KIo>G=`1Y5FfO0~Z4$mZoU2x>mtf!9bNK;_9%>e;|JA{uZUb!j~I^w`+* zG;dNWq;bMV^;5^di?kR}UJNo&sF#$9L@se}o2Y+3@Q1e^4rwb^TxIEB7{ghj`J(=_ zYA`n}Eexe}H-H}WaL|pFA(Sn$``NDK`)2H-X6+PJ#oL1JM}<<*a<{x^k@(nDcY}<4 z2KJz@&k)BmShlxwuAh;8K49F~!6!PoYx(dl<3ek&JM^wm(rk)D<1QqT;AuB;9Dx?kL_yg7^ z6taM!p^&E}rhlLiT7Q3$LK4%&gCVJdE(NZtr{ja%>I(H3K?msHQ2bQG7&i#RXoE($W!p_*KB4f&2VN(VIs}}fFxMcL4J0x}eJf@esGe}`09Cf(H94Bj( ze20!P=sOI1IPa{kL(J$;4zmr_4ztCDlB+S%HWc>>=lDc>Txg;#?kJ3M!xI=}dt3mr z)rt0@N0A0uoC8n`c;3;s%u~FCHrzOlvKyQgm_F|*~pxu6(~;K#Y$B0 z457$Vs3*doX4$nnA?mP^9OK6@>X^<3`v{-fMl=Hq!Xbk?{xEgI9+dVW^vc#Bn>&df z_OUm~e_fW}Ao$Yi^R>1yY^RH3Lh)g`lHJL~qE>f^PlZ@0s@qTYJBsK4(~yjUjQU0S>2@_G3fx!{7{T*^Fg+X1$- zjf{|u2N&2+rDDzLQ)<%oVH~k#OLA&XHB@lh129C=R)uz-JAvbc`vbs901FbwZ_sLjyuk=2s}TYbFzLOJHlW*9mtNTDJwo zQxS>e8ZCL$?i`W;c%A}NVT7wY)JXCqlk1mPQoE{)m<;UhZeEy_f4D~=iPG8~&V^rni)NBn>aFU<=IV7=3Guq?H zKdK4AU7MpRX;^qtY!SQn_`S%_*+n9y{EKI#Vf^6UQgxHN5g!&{3kjpEx|UtlTd}Kp z^96XSLW)3RUsa*8qFtf5%)4+EGVZpDt=ZJR&IE*m+R_liz6M@FJb5YXKcN)rumoYZ zyqFw?7GK!BQu1BgdJs`H25u-<;Vt%`xwf4EJLZrfh2*q*B?dVqz6QiX?1>`4~HaB6lpl3L$JJ*DZ4I~^v3 znlHa2Ib<Ni0-HSC4$ur#I5j<)M4ob>nACF{rPN<4qov29 zSLqQ$DlP$zhamuNf|+B}DC}X+IxzqrkXaz+kNoKj_E?2@5QEtpjr&F3o*gz#yx(&C zPE!#Eh5ASAahJZ&(MHOifmZZ*fJ3+Vj>KK)mRC;Fz7#gxZO!)YFC~|l%eo=&UpLSCr)AH4uxm<&qdVx(vt&ve(GqJ&Zf^L#1018H zNpY%wvNCLpXv%4CvC=?unrQYIg3%E87*Joel~se+s_tP3^y3!o55HCWNBV%-AGYdN z2pVhRYRoOg@Bm2%DF68xl1srCl?T)mgRQ$VQYb&`xI<}%vm7UJLsdb~l%3Y2@E_jIb|`%M zDW~Q~>;vIcI;q8yxw@F@R>GUWpyRj9cGM z%~2mRYvkO<6Fs}8n<$7=a}d&Dn_1W|I>OooT|ikADzgX1IyGCU&=-ZV9F7%;L(RsS zB_aKv54zjFn3{tnvuB@+DcPPGrEDGhGJOZLFJHb%a*#bM?^$mq*X^gAqsvOLbWNp= z1fKWO5e#1V*%1$xMVA#*M&neE)xT(F*d&Ez2<*X^PooVCuO4{3xp$bd>iA>Q{xS-0 z+p+x+jxb@^pqp|K;6mkCPbUjBMNFw|W90Iw%EH_MWaEQYFT#+LBTV=l*ZxfR`GrKm zXP8i7?21z}g=BElTZ0Fww+3=u4_WvF>~p<-5A`F_drZ&f6&8$en+X^4D;Fg8B>o zth`prld}W+{r(#<3B-p|uxzTW*lLT8a&a=L-ecQwY)~Dqqay9|b+q?C0lxwk#I0Q) z)ZBiF|JQWgbo1|BpMNhG|G}R8XWS|OC+?X44^2JfZ+)ly+t%g3^^5xY0fKGd9rHc; ze?TsbJJHWaZ`aY!pYXrSlmA0^%KzcI{Cn4zzn4q?-Az37bM1pWl>Y`D=yH#5^ZNWX zF8-OG{9nF3e?b%RIp%LKRAci=&?qS1*_|Iwd5t^Yf$}PM{xQn`8tkfXp!`4Fc~<_V zl!x=MR7#5TSJ13F4Ek3+=qdl4J6}%uB6t27$``ovrzxNA&TpoCvOCYpdQ z9csDd{}cX6p8SP7O#+q&dG-`GR`u6N46+a2>??8$#0JQH`Mp96HA_t4M5 z`uYb#)W6P?|KvO6pHr9rtor<)aPdDFp`07 zLAsF@YbJvf{MXMzjcC?fmBd)pA6X+Bq1|WC1&pki$eM_ZkA3QvA;^RAocEawtwhGj zRO1_(FLu@Y9yLz9=9YT!R#xLO`Up`vP>r`y13Ma7l;6=WS(I%en)StF{29*j?}ad-biQHC1y8{Kb8H8zMfGCgXn^QbXHtg%O|Q9TzmW&-7* zu*ncsR@g#O;%!7W3S>~&&#W&&M+jwP%!2kJ?V+$JkEVNgG#w?5?!9Q*5x{DyLOn>8 z9;D%-!WxKDK1yvJL2VBy7b7c6c~~TiBeIRG5!L%RS|iW$w_bHCY_!O@6q z_z>AFk#VL60nHxl0sf zDv|vHRYPGe61fZdZGd7KyrD z3sNX7oZ7OWhKItIZgwjyU+j#6$cBlGmv}Hv^kDpn$au8K__Nul(ORrg;8CN6M~wq$ zi0q_TV+GZy0TT~VUPM+NyI)1JDMYptcqr@yP?WNa$l8dE<2@LMdN5uo zGVUxgeu`?GWhhlgj~YL`;+DFxIKzC!8sn)(DQb9#5(x@flwBfOFCu#eWKdW)*6pA{ zgj&ej3hIrthr&LqbSrGKN7K<_)00Tm@uHOPdXVZpNZ%B7n*dTM?8pFfEpeO z3j~F%u;)aH!-y;uWKh_3ND41jIe30d?oQs`Eny{U*gGHDP5Y+)Ujaz|g81mE0qS{r z1w38CDLms9=5;-fqu*%Eh@=BaUUTq1Q9Rp5o3YpLdaIG&0nv1%qn zc&QU!a4J(5Z(;2Q*@1Iv9{3SL;5j7<=HbQ_jto4HQZf zSdu)RtHNnF2|HcsAgzZ(n%y;dBd-Sjdo)n9XkFdsEsxjD^TKd?fbr#YtkvSfygH%o z>u=<*3NN~CPqs%ZlCaI5o4*$p*mE8|&_K;znl~rZB^Bxv*bo zgtaZb8D-bJPi6MN_D;=%ROpMs_MAoKIP{jdsC4Ym4LQNJSWn>CKf+XL-fO0oXp};EFQz+2U+|Oiyvlj3X9WNoG#+e5{DGT zkP;5V8!wRt4#8g>wX^8-K>qVeNqnQAI?c+;4Z^`~dRUgCZwGeW3xdQtj{@;12`7Wa zYBUCp(a;arK@l|_+ljs#Qiw6#(9fWLKRzI2m^~1WP0K=!zLgB&2Q3#4vn#8H#a3F{ z8HB_@9&fTVG6;qM-*P_opxH06s64#(bRr&F)oI$L=b9aD#r|N_?=BrS4)5&7C@_v_ zlJ_m26HVX9wl?0@$cGB}P^<4IMMY#;-Em?s`RUM2pjkgwVE$n$b#N8%0v}MT;vr;6 zRM~l6y*Vhl9DILdj~2o!@F3+kmqcQM@fjuZ@prU@VrpADdLxgsx_%HD?3Iyawf+ug z8x9A#*)*!hg2@t9*9$6xJqqCQaWBJb>qESXb;^?#QsQbRd~M zaG^_1G(9JxlLp$9nqYLXe&6Iglc^hHZVASMJ--8!k|xlV*0>-(1!2*u%5ej^n{-5V zp`Tw+n^mbruVlXBGfnkY)eB%)T*Y3RR>xN?R4qW>VM7&1!Rs#Vg*f_RXrRgl;CF@e zcXrVqJzbTam+A}q_X8_VIHoS%L`&p~LcL%k5{?wq$MA+1pRk6ZNA0M=_K+VPp&s5v zc{R&h3NUgw^q?`nfG;oViQCSi>VaLcgesj_SXh|M*}}X^>R`#2sJoyJhA&^;L{;%I zw8Fgov@7x?XcF{3%3Or+uuPMn?SQ*g^AIc8@Z-_^HWkUd5Q>&n_b)??L#RvTR{wHOIl(_st*2G5CMi!r5CT#SZL>qukQo$B`uA-sKfDDQg^NoEjmZg($l z3kkzpZI)1BNT4yc+}sfF#QB!vr8hGS$jf7~r`t}Hs{cnvcm>M+Kx(YzCQbhSV;;v+Wg>OEu^=?nYEqLf>7%%Yrk*xGo} zKF4}g2iOL{v|F#5>p8ooXbIQ~$6!38G|mq(cgXtg<%_O5tqtk)gl|i{LZcQBzkh>r z#}9av9aV+nMCy2Kbm=rvecp2KFqt}oJBEXC-&U7@x45otV&5TrRFzZH^*ibZUi)i>gj4h3x0J2i-ayHfF%fxZ z=U|l7Kz&f_|owLMj1Sr zszH<&R4)^-sRBVxlzOCtf!d?`;RAd{A5!d5RU?HP3*gR5snP&s`N?Hq81yZ0{2rwO zR|n3G7lOWh4tSeUsg;p9HWBoRj6_960VFi6W`-yFPUqQ8dVu}fM$$Oiye673k&~pJ za8W(+P!iM=fDY<}N~WK;AhzR2bc=c&j$S`Y8i0PnCH-_{8d1$4U{jsjPW?ZOSk%dW zq(q$@ky3IuNhhBIC(t|Awh+Ab6x70uw#KaTl&^k2Zk1&N#P;ox!tmo|}TZpP6AvoHYPPrg&(U zBXr{_4WTFASqUpi8pAVbo9Y|X^X&Fl$fMhsLm>+Jyj3mL(}Ef?)r$cv=twwR>$J{t zyy9fntVBg4a7l@RvW&MW_tx%(+N$%>D{aYtZGzNOEM247Crb2Ke@@Lu-;iEDxQt%X zSyWtmKf{YwPR$d9$Bc>Oi!#EZQD&s0y&%9XFD*SBTE!;Uh=T{n-~!6nIyLt*cH`MC z;-4S4)hDx4d!iJ5ShX!AMHKu4Wh6=}dt3b1Q=%L(ZaHvz1f^~{@L2(R@xP87yHU-# zr^x9ub2E&H{x)X`Nkw|`ECr1k?t0nDNDQMTjNe{D531}z>A2uIHSdA(V&p!`quhNg z#~cFIR>0DC{+*f#H G64R{a21=NfYaJodD7?cb&c1vUVme}+?eeAFJy_^*lh z^L>2O=lJ@=QDm@3Sr1>zVt5wxkh~n$%W;4Z^wMnIQy%ajU`!15;R>?X*G`gPHdPUE zgh)ZP?`|XKjKlT{YKzHP?qmTa>&?NxDvJgE#8afNTik!3EbWiVaBq^p%5HOS0=l+$ zLU-G_b2n_LQ!|}u&K@);hjNWr&OLWNVAx+z$iog7(@r+fnsSt|IZ-VZ(%kt{#_bnz zG$m&7EHMo99jxjuiD@LzFae zj^m5mudr%=HaosPF3t`UQL@?bnK&gTufRO8^Zt$rl=!P=fXD}T>-63gyVt>7-Uhqw z)HFLmT?{&j4BgH9;kyEm&AkbcL~!jzP_d9z+zGOxPPF9nS$?}%P5V&IT`$+&;Cj}L zXSDM{E2s_%Xb64-pae}372sO&0gyId*SBB{>`O34d;&&Lm7pljro?IM`%$0mdTO@6 zRJScpjd`}aA6E{irZMUj#d6jrMU9-bJ-21hdYlJE*Pbnkby|LMwJHQ%1ikzcMo6Wg z3Palln=%;k&Owp9<@)5<&08*)2%sHBw9d{;U1YT1gSvAFkHcC>t4=RM|8f7Jngpci z96-%DqQF)VwHM+KX*o4!@IcKdH+~W@uEsnPsQq;*Yxgb}g`|t3{ufY!8!2lPTAqlO zu^=`9g)b1`v}u&;B+@P!b`X~$?0+GIFN6T&c|bWepB`nhtVISUPx#!a2_cL<$OMdB zV--*yQr(;+T?`WpxP9H5C>(N}Smt!zaPlz!JO;tTq zGYDIJLMg$lmBD!3-(X0A0yUSuu7r}j)1 zbc1(_SS!+a*i|zKi%-zl6qj}?EDhN~T1T%JaB02e>0i0;nV314a;AKN@$OQFZi1%ZrXK#65!keMM3K$2Ou>b@981_o&j4 zq3y6^>{8-ldBD*S3)m3`o{rinySB zk17yM>8?T_NwPrlZ>hfD05{FUZ+*0>LNC6X)G5wHNDX=uJ@`U zr>|A-YO@6Na+{VesQHsg6uNUg0%${CNp zZ8GP4uEmsXnDg^}rd(^ha9B7WRb{_Vkx0zE6%LT`_9aJAh>beCf$m2o z;L}gOakW>oQby#ydBr;jb_W&E?JM!5gqH5Jd>QNFN-D$+ZW3=Rre;VGhBTBeqgI(O zt9T*FX=M$FHDF*;VUdzF7#+ZoG^w1LKA&OyF2f^(67u=#d3(Zn_45^!=Bd$@AfVRt zPK|&hOi&=am;rX)*AcatjL!0~gVgysyGUgZ|Fx!o4L3%E`bFJ@BJf3>Nsbj51(tJa zazDj;Gx1^L6{QV9l&MajS0Co=q8jyNsaXJd!I7T$(B6W2w+yvuO!{sUpYa4(b$mLH zXFQdXTe*YVt=zn5+dOb>JSC`T+hrs;i&FYj4t5yz+LGrf_KEqVC1>Z+Xi3D%?c@6j zbE4A=a;mLuil&7st<42qYoAknHTws=jWI<%=L)BPQDMOa2J0le!jXgMJh*oPz?ie32;>wF$va zjSq4{86QFE-}ke$eVA6vU!r`17S)3J@@g#~g>+OxFdtRUZ^Iq}KXB^Ae95<&47BF7 zE?ETCu34!0LFDRhy1m#Hsn`lDd>bwaGtsVF!c3AaVLn}P=%-;7lm zNCa3Yp8pNDPIV7uamlz%M_i*JCNSok*Q27d^SXWVkO8*>xSGaTbwq_d@@vS8wg+?3 zF#$uy7X>ICE1qNtA2#SN&Y&-L(L-QDhD;Gm4!R(&&PFmaaNsLzpw&|X{E?&JeUe37qfUTi=St)AB#8bCH`|+JdefmS-g$KvHIeu~9~EH<+^12G;i@2?+j{p7m-2z&Eh z4R|%+)qqz6UJZCP;MIUv16~byHQ?2NR|8%Rcs1bFfL8ZYQU?3{~Qg_X)ydDWW2~VCmIL%%GA%73+1i@ zPbj5>V#3))!}RPVRLzjd^5I!mnPcTd9D^UmPS-wM)5S%ASLmoZ&dpN;LNGhgNGC|~ z#lQ9k>GX0!zAy?e?+r_JyZ{!9(Spv^^ncC}czf#AfL8-v4R|%+)qqz6UJZCP;MIUv z16~byHQ?2NR|75$(7SHlpH~B34R|%+)qqz6UJZCP;MIUv16~byHQ?2NR|8%R{GZnV zeRaqC`~Rc?T}!TzJ`&~qc{SkG!0j~o^Udr=;Yvum+KNvPU9pnt0&XRknn#LQ1quiLqUPbY{|;Y)pjn>W~OCj zn=-WF*?H+{79_HAwU$XHR$|RfC#rk3+4zVq6=Z_6ueNhWmxj0GA8)dV%rZ;`>E^6_ zstoc1nR?`)LE15C=G?5@iJoZAm^V>7HLXBvF{4EiqPxhxGyVt<{gSQe>81iPGZ-z1 zSdQylKy0+(oeR3$tDP|2V$$ZPSu7?q8tvD)V1P`f{}T{@}`)y1Nv)|6UQZ_ zJTN$AM1pb1s9{NN(!^XEhmq#Aocugeaa5#O_6PM7?~niU`n`LK^{u&+bMvO=x*AG| zkveiu9DXh|*E2dgDj_a18n9PfOdqK>S0CR?t|8M!_30JWGg`(%uYG#V zz$l={QUyj1y3utI5(9~^VQivd(5T!z%OrE&)D(-pz>=ZY->RM45eA$Q_uGe>h|m~P zcmF*~oR3=5v~&|Q3X&X)rFvI`Z(VOfT85TsD2*(MSV;?zX*SVv_|!sAV?Qn=kS_s#lPb zmm;f(wZ(ENG$X~P(f^cT2BYzTxRm(BWWyk10&G)~(GZs~EaCob4=QOTgi0;|Bzh{h&>viqev~i#1Nt;BpUdL9aXDN8;;BgI0;Y3$ zoS8ErRlrRLCY_svbT$`(I1)j}b>}jWcU>Ojrnt$dWkwAX%A!D9pDr;s%aR3MK5mN3 z%gsPKFL!jBIV){KwxlqLQ*yS+lz$URT(aEIn6$|zLw5E}6mq%NAVc zF4ZGVm5G=96TcIWzeE58@xRs2O%0okx@DllwfnX|vRy|#=*}ZuUlw9QJmoe#<)L@9)+c7?B*GeAkzzb+sE?#I0YD z@#WGP$9!nq|M3ZXPFSQ@1&S=D`dp6|MBO0;3wCthe+A3^Z^L&ktcfcg^4gp-#3ysbY!H`NK3>a@IFc`HAz(Fp%)&-UcHu6d6?M;b6lo=2EJ8UC0@ew-#!nn(DXaek3L zIz>Z_Bk&C){g}v#Kf>RN&g`ROa~boM%{y6sn&1E7IHpJF#|7DXR%09*bDEJsoaR|^ z96GRdAIO#6&vBs-a9sFB_gt-y6Kw?UKWHpq7aEQb$4ulAz?ETP{DxrhyZ#a!o8s*d#wL7=JFR6T(3czV&f(J$e2# zAp6@9{-9#;1m71X!?y-^*QGZ^$lAi6XY_%5_3O$!>Oj)vAK5M0*8i(eUL#EuT$2eN zax_qI4Jgm%d=S^rEUk?IB^O{;=KH4l=$QQSTDcvRTE^ONqmkVecB^mpN}kO1K6@wJ=I(* zbMxwu{AQ{0eCaqMi1 z4cyLY6s4RW<{jfH&zT@47u2%>7ZfR$C>r>14Ii>JPr7-{USMB3Pdy( z6vC0$Aw(lAg}(bBBq2Q+fhg0&i1{PZ^LUxRR>I#~4<9Yz zch1dc2gz%hxOWBDB)Yw!^`de9Z<3pg`T_*JghyO`GwOd{{>sQOtvFEv@%A7 z^0?Q<{^#bZzpXnjyAL#s##sy;tH3c9@t8;*<=l$ib=Kn!=`GXqO)2yQK(DXE+nf2% z(*WLg;dUULgO4fcUXI(0kbfTc%7Dia7GJ=#I>71Ous=h(1n_-?Wk~D0V$<5W`&V5G|c!-t|CXbJ&8MF_nJ836|vL67u7K=m*{^f3vr7@-tt4!)Z~ z2uG1l1ze3#hP0>uhI~8I5cJ?f@F|3`#1pVj6301+C*Vqi%TyOo_T>=lI~;unJ;4No zUWa0UqG0K^d`XH5R#A%9?NlK5Gs+*2Yer)1nGl- zEgl39q;-Jz;e6pH@b~l~b%KBC8HA(26MPxrBGS)K|1bi1E8|nRk*AbpaI^a>v2ZZHFhXLjwj6}K+ zaO`C03hBjwU9vGBkWK=$AQ*`T@GXS#NS6Zo=b%qWrvgqz2tax!;3kCYNS6UVlgn{m zAYBP~0%1SW)qpedIPOEF*8skg&vCnuz6@x33}b|JA>hvl=a9xbLpb*d`7S{Y3&+i} zqR&Ww0eBrD3F&|-=s!Xr((!<65JHgt5U|r!=n3g0z!HQoq$>djPJ=Fx9uGJjVKLH$ zfF}?ZA{{&(YYkx;(qjRO5SAmo7_b^4`ZtUjU=aL1Yk(gK_$|VDqyuJhTrB)JdeCeF z{3n71>Fa<~XM;bb0SC{)cp<$E@G!!1q|X5kmGR8AQV{|!i$2N5Z({90x8(Qgir|3EugH(%zp}1F{C*NmsDsG6~L=V73HDNKh}qdnMRLf=UTK`>@Dor35!gaF+y6O7NTn zuS>A;BVxUF5{#7KUZ@1TRTY z*2{l4%#`|Xk>KAYPhg6u&r4D}y`}svDZfdAuS?J+1pHVxZW(aPhJ(%nmreQ{q2P4m+;q(` zSEnT-Jmkhj16MW@c@Nw;;CSHjA9T|_4qPet9rVDJ02c~=xB7XnTcTdP*DX;G-s_gQ zO1#&tTUm=g#(A$>qHg}DU$rKAETb17QzF*^8QAuw-Ho8@r%t4}Q;+EG{BFgQ0j>=wQ&GzL@Y591!4L$n8=RT7q z$(%>8WD}O-A_)^`#jDY|asjtpF*w^=FbS_XoNxu&TplNeXC_}~KX z-}W|)9XoF9IQYGiN5>^a^pxK79mJt)_zxckmt)FF&xdRSC{E3BDB2$E6hHo6@r~v(jbR;BS)n;C1($ ztX#Y_&(T}}gTZ*6+$1W-9EEc}GM)T@szIr-dLF$M9TqCPX+(_)Icgcj29#&b$m-h>o-ixqEdW@ggh(^m|b7P9it z`<$G-+>|Mq`R1%#OD0OW+{~AhZfyX#QkU6pBB zmK2Nl;%7e&@1{(IlF*J+BEO?cqiC9DO1D}}a*RBy3$8bYp*JDp{Tx(%48*+FmRmmpH9~SRYjg(h} zWTW7(^i3>~GI{2~rZkc(2_}K2!1G*Oc3y$BV7NpDgff}q%$PMW;K?vR?9KCPU!y5) zis>f&d|!`uhwD}{%@sBv)rej$?_#z(qLEf$;gi3 z@|DbZCYj85Wj!Y?Hyz^wxlCp*ifHTF?!*kPj|UbO$&{7>B-hYwyGLT`8455YCQ5OO zJoJ!k#G1i{0TsMIuList_;)o>`0rNq*7IuMf3pUHr1N8PAHJw}{!GSu;eUb#e7InA zQ;Wyup@7kV03JW_+W|?p!25s-Hz9={#7K(1xq3*3Z=l3;cccQIQ*h^ek&u4*&nUR1 zVl^4{e+vHe!H#|dr{!d8rK!aK3&QKV>Ufz zMJuL^By-jjY*Z(j3S5Pr#R=?24&64IrkJv|*%bHhm{yROJ0)+j$=p$E%`&j>&-L%f zJ`~q+K))U?-gPUx_CG46u|qr!AR5ZBjr!Oi}U)7KSOXLY$0YiCH~P`@cbDuzZKwT!IzQf`$`kREeqdA zGJ$h0mx*sJ;fxZa(c#NV;=i8U1f+2Gg=>nsaiGb;mzvT*KONGfA!lMu5C;qU+LD%0 znL$P0Zb}0-3n|>tb5G(sObS<<@t`ii7oA9!d=Dy)QJXje-i1ws4H${_Oy7V?!#9yc zskJ!hW@VClVzN;$9p8N_K%H=j$4F2P;<^Bn4f+X89y26||B7qiA0L3DV6IG zqVU})`o=vu&k0;@}3+9EBdV8qNwO+)Y9U)Hj4_kdp#E6IRF>5$2D zj?gUlYsZy?o@gBFwpo8PZ#9-$^k1x(#Z5s^Y3{mb6$h=0Yb77}bZD3~<5?ch%0~ll zc4>^pSX@Oo3x@TFlUpdIwNilE?y*Awew&EE`;#?r5a$As4ql~SvuR`L#ZVsqUp@L-^=ppT&c8;L72J3NCBY@3CEAkk5?x7jiN1s{ zNh%pzl3!9-vbdzAWK&65$-$E95=Y795^h!Ss?b&8t8}aMtN2x8SEa7XUo~wN@h@5R z#;Vd)WvdF&M)j-US*YLwCEf(QKd%P78u)uOu&^8W`uK3Vx!q>^HMnO!KmWHNymwqO zw_6TSSw0FygjyHm*C2I1uQ2&4eSB1$Zq7o#hF$&eX7SuUJ_=>Yxg8x{XY%)*D!ZSQwyB0x_5xQQHy~NM=jzcy&A~Fj9d&TsM8K)av zSGZ7JbJkijHS>;7%yYHXP1lvR)htXdNwQAJ&PvzD(U+2w@yT1}Q^D+EE#1>F#Xa$>-TcTQ~GBUVq;gou27B;N%nE zMwATCTR-oa??xBcY z&pEm4zAq`d`}j*iU6X2tKD4DYqxYIwL;An@*js5^3byoX6!_DLs4@TexM!d5UTFRB zjrL9Z&-I=1!?l5#ho6tgx|*}4=lb%;r=5H@=|aYyb={P2RXlpOy2skJCv(4YIFEOD zOTS~=k3lPUA6xn2PrFaGe=~5(mbeCct&yfr`ow)mgL)=A80=N{gAr%>h2Fj zF18Gwd|$6-Q&mw9q<1gvc6x_tt#Wec(Z{at54ezHdog(P+b0Ix`~AAmO}qUS)`_Qb zYSPo&^`4z&`*>Z#fZTB@Pp^7t-wNG)ox^vFes_>M`oP3Dk{4br)tRp5?K#tSbCD%$ z!1`U2VmJRVG3uqQ0o8>nUjO5zxfz4s__bh@-;xX2rdM{1dZ^9vF|GDY8$2_A-Se&I zwphQQW@}aBIc@Yunz!h3ef;v!u4fZ|{pq7GCq5AVi0a41#>da>zIgEZ)dK?}2RGC9 zUZgzx{yP7`%Y&Zl+aUABvE#eEx-IZ^U39;LgM0UyxqSPN6<=OyIOXarRcrm;XM5Fr z`S_vtW^)_fT)gg+*^1@Co10(XpBs8IZ^G}dC#@}=QMy7i@ON#XwX);5W=mQw=4x7f zxpV%{-#v9M<>EhluwWIh&Q&zk%~dpJ)3|LXC1zh?NJ*21odzuZ+5g`TsGOZgt*`a${H-p`D0QTzj2G;P%ig;S}-nw+mw6L~Yr zt8~5qx@LsvtPIpO^b3F?QK|fWeRZt~sa1yPn(F-glt^;A06)Kpa{^&#ba7PJUfECA zR~KET-9e}8DYvFjD8qGk>B8i^ zPBA}J;+L0RkY9(P=emLY@SUqra7}lt`bn_7v@E_fYw&w-g*6OZvuI*z-#I%H7CkY- z{|$Tl`#TiJygK=jZrb@ff=f3{k4--C?zN-wANw5}pFZ&mee(tJ^Ue1hiPZesv}os* z(+@Y%PrNkfZu2LdzV(T2Wz>FqLh;*y!4KTEdS+(X@ZMW(J|p*BAA2pE`|uXkUv zDloZA`@EwqUw>fbZ|fVJeR%F}VPoOikLK_r=0zNjJg52obz}eRn1U}F6~}0+i(5@P zy_S!D^HA^3Q_Jlwn}4#=W2>nQvf|o64|C zR(p1m*;B>Su&0VeK64dwxV-~4oktWL8GB>>><4{&gxYm5dig_PyS4_xs$lAAP_6otGB|yxRMnaQm{Oi$|THwKuq|CNOc>AU>E|aq7v0q=8SCY*@e7+?%th;@*#scwt9imQRde9ud*z z?FV1?3%y}|bfbCo<12gBtbK>yzj@u4?a!>dUNvXe;m}y)`K*^$ULUeJe|Tz5&ZTeL zzsEmmT#&v$mTxe|(JcPh?to!$>dKoOoac9V&JSr>hwqQv?eq58t50OzTk-LR;x&8c zG&$^hx6f;(pT4i`w8wv0j^E0PxYmzfC>WoTKjhWlujD%>1vGs0%ole*bU9=ApnGdS zdSPk#jwO}Dbh;77;8U9~#*Uuz)vAlXT*-fT!pM1+GR1QL!P*A| z%$_``b@J!Wz3&$F7eXvCcWZ3Lu!K%4>Vg;~Jqa?7xQIl9TZcsZ~0@!)Fd{;zPgxcdV;%KC%u zFgxnO#&fxdW^@kM=kWOxhs>w``S`d-CA~h~{dmXfW$y;o?uuOdh<5zww8wHDIqnm4 zYM$^x%pu>Eo4+&m|LOaNkrP7pem(S3WN7)!f~4p7Zay>SEAypcC$G&?hfGKvGVzJw zXU}wffnRg?yxr@3|8eZX4>Kwk3|x7rqb=qj*Id0WpxAWs%!UcG_HS96-z=xagYo9% zi}$oSW*O>l{UP;Sw=GLbrgd34ytJk9=~ZR-f2e5o&fF;v#44x#vS7iUq@SC9J8y0G zb0IfMo1LQ>ME>9w^FWPP}zQNKw+k8RNXUTO7tynJ ztCm<^?R!M?%u(&$9Zhr#-&+~rla(=Xpnm3aLpHtr<&4L+EdBlQW%k5TDGMu4{BzLe zfQJTFg}r?G+VC#I6W{$X}ee|dy#~xd|<*q4NXVowLy6DQHdu=n4 z=WqM4QyH&nKGfN(G74Z63BWy~m*)*H2A36@1sX5z|KWJ#zfzpMG;Z+<$hr zoK5XJ&A2$~@Px3R&Cd7EFRH2dG4-+DUoDSX|8|EjJ1=;k*@*!^ge4qPf4(&T#g@-4 z_uY15-I??Q-%UGtdf@%f79Tsb?ucp3E3AsJ>@Bhmdil>j(V^s_+E-?cX?f$B{;wv-PJVXcn>SXUxb|dX_UrfUy|jum zP8q-I_^kUDZH<2N%}?4pQwRSxOMm9;!S6hn_mFJDGho6~>zMFc?m69C(py@1x{veR zk~XD0GvNU`KWs^H4|Hoo{-?Xrp`vAu*TFWIz&6kC`!8B%x_T^?yzTmE%nJ43hhtuQZc8V#vUT9S zkH0kQLWlMDzP@D5N56dEr6l;BdtQ3r;F*Zj`$isamHlXmp~G`;tKRsopz*AV3oD!d za&+eA*-MUed1?5^AAKFMZrqB`znRdm6{Nz6iUpmqHwFfWuZ8rYF_ln|E ze%sXQ@h42DKihOt_i;y~>eM^Ugwa&z!i{F>l^K@666c zFaCV~F2VGV%o)q)br0%)*SLPlJs%#9djH(7{T}c0{lIU2?Ehx$$PbnGuGdHG%>LB4 z(zf=krOWb5)>}(01${d7&1?N^mwj(9+WUI=$M@e~Q}*%a_NKxkL+*Nc@vkE$W?wry zVeNkZW4pS{{`&a)*0}s{OZ#lrmCQ999=zu4^IMw!7*Jc>ch1YP@AddP^o!T*H=auD za%kf_OFwE7GF=`*>>{?YgLy8200_Vv^=BTuZnr`PlCr+&U;biW6WPOAMTlp7s9FyWW2E#Lr)?JlgfBDgVlnRc}uU zynl9&FT;;*+BUgx!Lc?Gb6Y>Eo7*}QE+E~czh@8c6MU;%2M<>hd`m4|?wba_5rH0V zCZ(>8N3KysW9-Y^ZmH(Ff$oaRh)AW*sqeL*#e3~{M8Ey@l`pG4zw-JAoZ9eSkY7%h z77zZv=FU1Qs==<4g&`iaZy`AHHjfw38TD0q) zS1ghHv5khQcW~MvVLNe|dfW0~gu&z3u_v9RZ@?(Da`My4Ae)@s8-el1ms#38AV$rT z4B@^*j)Y155o(*siy!lpZo7`YyL75!$i~bV>WVh&iGdQ@co)|5SU*(gm&I;?F?$W%zDBARqp~lzBj$%-s+bPY!T`rFcw`@Wr&RCqujGjxti7RfEsH`IL zkxEky7@X3=C@F#zcx1lUcB%@h?I=nRtc%3DMix z2UW~P>4Ft<+u}ZY9XtzN9W94t$yk!7CRgAkH;#4 z#7mxpR%VEQ6~y4uUobP(5JV={5l<6u{<3X}DAkgri`6_UkbDdhNdE4Z`;3N$$`?ou z>_y3&f4SRd@RGa?I8_YT?cafy`{>x;c_1b}Kri7dVE7UQJ`X(R--oT>JA35$&hG#D zqj+}z=_07ZB6+4mL-w`N1GEmfY#!j*c`A1Bz?Qlr4NCB-89>Co;@9SKK4MR=YQBBk zlvO2hKTH~(9FHHuf97X|_=Ew2BVd&2nd=U5yRbP0C?pw#1dtLi1`)14KpyD~o!#vJ zWozeV&G*Ly5CY*j`v@8l>v!8v_niQqg+%FqM5-f@h`s#{1zNkEZJ09yiNJv4vjiMp z@6V2p040}Q+=&J`mM@4AT;i)y!*ubPKZB*ZKTh+HXTt^I_<1&re>$5BTX!Mgl3e=( z$M=75b^a2OzI&Lk>y)eGy#vKp^(MS5F&z)rxouILQFg1sTz(_NKQ>yJh-bj0q+crA zw75lGr=L1S@l=_CKhnQR6P>jB%J}Kr5FVR^BEbwbq2G}DEh@F(_lw9lW5|v zy`n{{qE9C2rRghzWh_!L(F~p%6{6@BQ&Aj!SWld-JV7T4P@c%icVmbQ3G!$WGz8IX zsD5s$6Qpv^*6k;;kvOf1OEa;W?#u}oRKHm)U^r4Nk!GMkxLp`qTWZyd2Yp=(AS?t9!6*6CV`ArXEP!rZ` zpx^jT*R;E(7EVO~DiI;RpP&IT#ZpqK9=y?4el&yVr)*FkozK)~vB-g26}m{Au8F0q zy`~-Jr-Gv)UtcE=bl>lW&4mbekk6~vxh=$75SfyT-rTm^ za^{G4GB_B7TFJ!c+pPEyPUO(P>tpX~&?d}VnFlkwvkx0m?ei!#X`oo=qeGY#hCY?G z*fYAIpviq4g_lB04|-Oz(Kws-x!-c}-U|>i?qMHGSIvncr2mRYsqT@=VLeo|pKkZe zx+8GzN2AUl&kug|ropBfpKrFs^+@f2mz5TeDYE%fFCc{wi-;_aYgTYJ^Q=_9HxVaX zpurnIA_er8-ynrE>QBvl223v^QVhr+QU#3vlq!IJ9*g!Hp^(aiW{M+3|BiP(D=z1? z*@q?fXJU81J?D|VgJOs{@$?vJy!t1C36vOTJ(q3g*;I0u=(ZGepMp5 zxQCGyJnK!LQiaBNVqND<5XrsyBq@|5)7#T2!&o&6%gsM=Z}EvZG)mGcS671`CMM9- z7vs{Z!y_2T1+w}WJO`+1zAD}c#)ie0JrU-+bw}R}(8XInvNF8FYDQHqUG9k{!F;+z~2}G_2%iH^Q%W3M(qTn~v+IFUrD0Z1UM*{%qXl<<#8Rxq|Xvq8P&f zl_7=%q8)$d3cifbCw1%+@nX6qO?JB$rG6_i`hQnscpzLqiwyIhip(!vufGBd|B$-< z66H93XF8sk_HiN+Xkj|i5bdFgp2HvuZS!4H*u&+86S>I9YOC($sYvv$|L z^C}EfWfzDOMDhF>sLH<%{5Pxe`N01hqChxym#zd?$N(oDPR(p^qQ-sOy|l}cg?>*L zX4jdefbcZ7y%{1~A+2_sM*UXoS0z>mPNxOcT55NJ^4^7_s(ZN1V)6e-kc6)7(|Z-B_|UFqmW=f+BBoQc zdq*k*bT^_4HELLb)G%I+Pc?b+SP+(z=;m-J>&;4C^Nk)E3<|BDmVdg6ES`vIqc*VC zMxDeC0P*2xnhDcNOK+NAubUr%NFc^ z)t91QCZ(V*D&AGmQ?l60PM9!%iauBUF&4Kgcf6S++oEnW%%@PSz+ZZc^`Qt&dsZWb z|H|+IU9@c@6-%K_2CX87g#3|N%WO4&6$!4F z9We|;($jnrkR?NJy|3sFt<~9~A8&ensqVeZozqN)$v8`5Kcl&nyvz}{pDH8X8AoG5 zxnQIWlk6(tW8D+}f^WYq+Gvt~9|Y(^Ah7$m^IaE&+sYd6T@_o&sYD9*^?2C+Ru|wQ z7lh$L2agC#!`h2Z1xPeo`|ttn;xkRSOoaXO2|8IIn9N55umRvK77+;y_y$kjR)cnS>Ga!>hEYQGT)-b6gUT$PM8+>1E5aCpg+s~bbdjP6Y= zGjE>0LE#f8ON1Ohwju$y^)Ra#^|R!Dg+a0Oq)-XpSuUb%ziab(BRvyYU1i&I0rsHq z`iDcs92^+n<2v$JcR6gyQ#NY-k;QBUQ=QM~wOAM93`65FHW8?IX(YUxjuWn+7JJ2} z6u+qRYHUY}*gCSaIa`GGaB1!GzY?MCT41u|Zbf(ZZs5scwgC=~c$EpL*n# zb|K&s%NiEbwz-NjhU70v)BU}vT5TP@&%@l;odz_AkO`1>{_ruBfKJdJf7+2x;)e5% zo*84>HOSiao-zvD*cv=;HI7FwZB^y=jx5#sKc7ZEvDm-yCNH@)mdr7nh`~U}Jh%ia zsjkwDBlKT|3T@AbpQSVpdnl$ZPLI1W`;X9R1g?8v9l`+ditZY}q$8Q6wi{W-zF$U$p05QR-}!*mmeO zdwT29sO1Vv53hBO&)q2XddTG$i!q7q&v9oQ=kT>KVo#S8Zf9Jf8N6%tg%+Qcj)}K) z>Rii60WBl^LCbV}nvI@us=S5;%#WwGJlUy0v+28_Wxp$Rztyt86WsV+A=CU>$dn<9 z5cy2mOc{jK1tIe|`zQ9ZCHJk@`O5+X*59we4Pg2GviiQ!aiMeZ0>q3R!U|zNKL&b^ z6yFE`)^q$}^xu>eAoPTag4<**vuP_=yPVZAOc1_dzoQ11>RW(4}TD%eak`WGo6}rb_&gmS!gN0qZIkE{^Y6)W}lRKgKQd^ zuAYfAF5jZKK2K|9VJw3JjpdlBj&b?OFt&rt^U|9xt zi}>gVu}qByQ?w^5@zuNbRd5ESn1Ro;s>7$u8~h37lf(5|`Xf>w z9XUR9R%7CA&LXojH@HllR@rJal3J|C!g*ktHqK(3%Vs9hhg&a@!=hr^djW{ULIz)3FGsfDDDNWnih+`HLI`tWnx*JwUgs#f2Vq^BzZN0zVH_oG zl^t`0ZDDRFQ5P!37YL38R4D2U^4lN`O`}zBT>Uy@t6$TbX!8Rb_Z`coVW z;pN7TyXZ$4BI`RlUJ~0qsF1fm$$D$-RNit3lxn7+^y1Eccw;EDL#>`yc{sShv$)t^ z9!J?&AgS7Jki_l93A%@_(pssWH@Cd}OoS3~(JQ zGYqhv#Dy2C-)d5qSreFK!xd63sfyHPgE zV=^9MSr=8^&rdWPvz{RP@P0W(>dnY^y~)qZ-wkkI!LQsuCMDi`4{`17+36ZMAmVQ$ zQA4+chR7sh>#1FT?}yJu__iam$^9-#J^Dk>(ET}ln$f&&ZnacIG>${$V>;c-mI)i~{{vm_1qc8D literal 0 HcmV?d00001 diff --git a/sdk/libraries/x64/tbb.lib b/sdk/libraries/x64/tbb.lib index 1ad9235caaa96e0b747ecae6d405714668ec56bb..ace282a73b612c7040d013e75eb3f194d40ee3c0 100644 GIT binary patch literal 47260 zcmdsAX^><^wLT2{h&loyir}CK1Ip0V(=**Yz*KcJBeT&A!)$Ecsjj-+Q&L^k)WQs? zfFL8Ah=9l<$S$&ph(P$mF#M~5M7%#nJR_bz48kD%=?{PSf_F~l&CI)HR_498I^KH| zai(rp-8}i_$?e>; zo9L-mIsK##K=hqc19~1mk!~O5bnA_X1L02KFFq*(mq&M+Q-3gRSswZ-X&I{=6tM7a>Xbqc4~Rv=%b zyOwc!^icq!`~Jx3u@RI9($9OG?!Y!NLVR~1JrmHQSf1$h{hS^^IS~EoFsIw_ljsS& zhv)}$obE>~L=T?G>49Ug9MVG@IlX{wAbPmN=`fZfdgyPQUcmc^zI!{TJI@0kdU1x+ zo$uq{NWZv;)6E|O5dG|FPH$p=6WtG-9!LB{KgRYF9sWC~`>@?aKf=BwdiY65a9vr+Zi8ctU#Xx14UC#(I&Sy_wUmZv!BD?rcuaPhdYHJ@-1NcizVFg!Jxe zBG21$*^bM1?(#Nny7H2pTXyWae5co&oh^-m=gsWizCHOf+{%9EzwX++duE%racb8T zl&jsY-|02lEpM*cX!bjPrBkZZ{T}{N!6KE)Ra5(RY?|JjN!GJtU8pMhV}wEX2}WVGMweoI6ynlU zulA}Q4kfQzcWEmhhUP&?OI;Mh}A zKTq4pQp}|e&evg96>g8)bl_twE_8-ABctf3UmVJ)5A6(W?G;0Qc(vi4SUcc*)n;S9 z<=64E*{%wg^x(kOP+7t}xYBpNF*KORK(j0G_QYUy*2Hjj1 zc)bOGq1{Q*9^xXhFHlyE(sGXp#i2yG(Iu^Pf#LD`T|Z^9_GXkkY;TtmNgpj!o2F1A zd%9Gw*Pi``U+Z~`?RGOzM!aoPo2GaBhx}Tft+lk7RMtlH4nmf?m7ze6 zWi2u``K*9>Giods#jcDxqF?Q%>|K0O2ni>SQGv@d_ZR0oh%!EhubSd>p&RSkm~>U|=H^qbw-2_8!BPOH@H(L#8fv+S ziN$`!D2{gov2K7Afw(ZEk=XmjcDw~%|2n{rY8o{K)hbZ7k8m`61ShZYRy-6 z%snxMfK0-}t3>t`GtA-Bcsg+UHGr7x7?+~u?qc;|nhr~gT&NcDC$|K9wKfXsP%rir zRS2&F5M8Q9PJT$ltjr%C(R$IUr>d>!G7%CAsgQa1)h(9%Rm05{bJyd7lQBHdR2qYPik5be%5>_YMp#i_4p zm8s%FRAswKcugA)bGd$qv6Z^+rHGNsIJuZ)=2LF88odUZ`_hbJa<;_Qllb_`#RIJd z=rU-h*w1XM&(44>VSMJRDN?KAe$;HH8^y%(F-RZf#UA>_y}3mXh0{Y@MVf+Q*<-~pD=$!s+2NI;A)bZXbtD5gdxC`nu`yv1q=+>Iiq}Kxl!tsVJ_>_e31c4h|=r z(i2q_U$PqLT%a^(rXv$$pd1V0R-SLRXRA$ITw7>PX?oRKuW=)q%&W~b-PhX|+IcdR z4C0$L6EVs#@qEi;qaDi*uoDfa(rhEr6((PW1}4+K)egaf=Myc>Tpsn6mc}XZYg2`z%A2i0OMwQG9X>!9VJu|SDIxW1F?ojmqk zbV~*?59WKqzv|10UovE{PKpfH24H?=A9L{z0{r6;z`t+BJi(U%Ry~6GeVBWA=zEyg z_XOq$J`J$-8N3-k&wdiGe~S0x=c|~nS^o*Zso%joKcrDCb0hM2>t2ArJcfVYg878s z258;`@ZQ}3|HM4NGcf;efa#Lx9_;P2lAs9;Xz2j2piK&re6F#CPXm;5EbU-9=p z<8Qxz4d6J$_0(N>{XD>L5Z?ym_ueZ2EzGmr@k7M&Aiyba0j$7#m*IVs@A=33ur9>W z`#IKi2l6Arb}Yl+_9K>;9>6@vUm@Lw-(frFet>-L2RIRV9fQAZdqjlGq4vff#cy*@NxJgtbk9z39uYK1{XsO ze3*whI3F&9EwBT=4!husa4xKeufWl89gIK;*1~yEhR?%JSP84(9QZQqhA9|>D%9b0 z_yU{(2cQ8*z_R~W?Ku)NP${mssJ*Y~C-(vT;t{ZV6M$f(ed1($67h=#ty!?tJtSsf zX-vdHETmHDQ!7yWkh*Acd!XH6APch2DSLf5J414g4HmPOt4a;bpK;A=&g!{ZiX0}( zR3yC3HaEFICw)Oa^@!m&w`n@UEvMBthpAM#0Ca8?_Sl`xFyfjnox~ID!DLo zh-DyT=ovM+7O}Naw~*07#Z>aVEDVcT<|p7#LEVZ*kD{dQXS8JvD8~9A%>cq`%pcib zyoGkyzHoc~w0&wE>IJ!(i;Yha+=d+Mn$yicxVR~L6-yW3OV8l>%c8o4vFH#`>+I^H+(?B4T=CNkhJTXGQk|L*t!JoblCY}&51f1v z;2g?PCXBc9nK5j`9oo4h3=^v&J+Dl6DiUc+c_VOz9XI0j$>^aVt9;VXo1!@@?O0SV zP1T;!z?jH4e%yQZ0Yv&JaC{7^rWXq^*w&a9|R*bJ0+-hmc~`;taAS z8(__uvL^GNLzl}(T|g=uNv)t9wjm`P6_FH+b3I#Ve~!aAh7zG>!!$m#Qq8^8@1Zz1 z&m6UsGTp4W>deSI5X6OR{sRJv4*JTJ*I+2boWm3XMR~70{DWkiVj2k|`SPCjG(TUxi0E!?~#9MAIUjP=crK=ow~v)a=WtGc_FVY7Qc`T24~f zX_pjtA5ygS+6cz%jB~rF1j2lb^tRm?e(J^ z`0A6B(ZHot@wZt_P0^;h^;Edbq-0Eo1q>Kl<{HC%YQ7VmVhr$-%|v!AijRSXo@cB%K~=3=j{sNj)A!Lr!CDWhas@>7g=) z(_U~!Cj6A~QL3VJG+DJKr<>@OB3y3)FZT?}a6vmaoGejH)-VDQoPnt3_Ufe>;eCf9Q1>&td3zSfVR7{?}WDqc0C zLuj?Ggsi-CyD5X-F4$Qaa5Z=znO>8yFesrJiaI+kLug{Qw~TQ{)_K?Ya&HCh%Z#ILZ)=BqNiYb`%Z*|?D?a!ZeCI1UBGZAxH#UH_E1y)S2Zcv8SS)<~olkwsYNdFalaU_0L zVB~kfed~g+F)tF}zmdd!^nC?Z{%#q5uD~2h`ggo+)H!g{ievGc_u2P=nUyQxmBxve zHENx9w>{TexoYRRE4Oc%*%tgB`8WapA{az~sh@PU0{=gnUCBD2Qfn_PBF_d2mf)T-x(5qk+Gg<0TJ;Tme+K;*{3GswNu=uEGE7PZ zbbJ-{qR-{Bp_U_LbZjCtLYA=+f=RVZ5Ck)Agw*MW(7(_KStGut5%Qe)Blx5dGK8MD z>Cw{>g7&4U&N2($aJ+J*npOXa74Ys&mziN!trcRTSxK`h{I3EPF{{SJ_jKQXh(Cf) zy02i_+Q{`ZtiGVNsx+cTBH%d0%=yE#UhQKoJgVRfV%g^)uPkRwsU+TYO>ekXIIl*W zN8rLEQh<|MSGIrr}Y^FGJZL&CuoxjX_N`k1R*i=Mn8L#}t50?`>9~ z<(1{!pBPJ*XazAJ8^#>dq0PDL5`_|{C7y9Hd15_GTgq_}JUWNCH1sME&U1KNGtMRv z#qkV`)3-FEN4ivg+eJv^2)M@Cd43*_0UC#=wpN^8YYE_6@%W8DiJu3 zmWfijbS_3LCnT|`XpEGtjK)pA`M+*59Zd<*d|aU!6bGin;XZB+v*!~Wfo*VE4iu#k z^^6u?rFWwoKFLuO6|t5Jr5HTXz2pi+xq_oCA`C7ULcRX@DTc7f*g6GNURkaPrv5%L zjx%4po`eR@b7Mj?J2zo}pQKP`2?ET4YFx9Ll40Uxj>;;sP(hGamc3`v0O-W2YzKBo z6wYhDQ^$EX{y2gN}lr8&@*3{MWH6@YFiPYxO#`4a5m z&qp!aJUgh^*Bo*j*F4RTmGkr%Cf)Oc2&h#svji`I`gPq-Lcj1M*6SD*u^03ZA^O?8Q=vpP5NE(XbYKEuuR!g083#Wr@&v&n%6K zttyK38;JFbgmuVa&)JbrYP7P;5&4&#kQ<`16+GvCqcoy_N;r$-DOA0w%9)``n__JjKKIak z1;%-;F=;Pfk!bAV{S^#(W!Yz@r=G8J6ghJUBqI0z5=&idoNvT&em-H#oR^RTlh}G= zn8-rAI>6){UWd63*%6gSg#sE~X&Z>>0)c3F`4bAb^9+z~=$I@wAo3B0+&w8n!I4*% zGr?prDiIpHJdLz?CkZ9PE@W8iN9|E*vPTp!d1WcBQYl?ix(54h4Z~INOMfLmcNvZg zsIhTI!5(7)R%B04{Oy{D(NXoJ_T%p<8uMAY!F!3ll;(<#&V!H=~3kfdbhC|S-70%{G`}CVmiQ9 zlxc-fAo9vmW+t!UC8;>f zxqY&J=N-|6WMeuKYBJJt z?k_^=iR~|BlilWqFCMoX0VYLgITI#@aVUn1a!|Ir4bQgh+!k2@?{)b zwmexWB=;T86oEMtVCpZVovDMm^X^Fah1a`j(XY^3JR@R~;B0$Pv*%J=%H@P8cS5g3 zlav`KhT&kqIpU zTlRISl#X&gqP!}GQk$#foWU+pI*&~g-mBwyO@{NSJtW+i-cr1LO&b3-gg18HG!-#Cb%H zi_I~%XH!!CtVCjYC`29{n&>FX+^b0(Rwqs5K(S6NY%yu9LKJmGVLrY>RV=S8N9$Tm Ry>wWDAFEd!i6QHN{{f`@=BNMw literal 102378 zcmeHw50qp@b$2~Lz!eb{5fyMjL==SS+1Z_4RJvzbVE@1ZyUXr^z@umS%}%pD-EDWz z>@K1pDk35pW1=y}s3D0l#t>tSF(xs_7?T)dj7f|!i800~F~;P}$I0=_G2ic2y?Rye zy{dYzUiauZ-}&}X%uILHy}!Cub?erjd+Xd6HwOz#m#@2YefnQ*<7{p7#@U(K4e8IP z;y*V`Z=4pt-@cBq;9HD+{~at~A7(-D4HmrNE|DHQ$O1}l+$_=?9>wQK4}M0ZH@+XA zBR%w1k=_JSJgwp<(nDv7^n|amfYK>njng82BAwn5>6u4aK&gF)NH3pf0j1NxhtiY# zB0Xgt3wQ$0ry!jT`jno%Cr6%AaKIP%ds#r~sST0N{Rj&vo%vmnmc9TwNKgN~ zNOyjX1(eR57wLS|jnXmjqqOe7M7kUGqjdCCkyajK0j2vPKT0QlNu<$s7En5Cy-2SB zUrNuoSEPH9kJ77`M7kgCMCpOkM0)*wEZ_;By`DaQP^8nri_&>0OX)cskzR>kp^e6fKqp#NIl4! zQs+^To-jgxMLKa&q;+RNUX)M|N>4()C_VEykuEry1(aU?m`Hb_9+Zv(FQxl{lhWPD zPia$2q$hrd1(ePKR!Vp77wL82P3biciS*i!uz;rv@e}F3Z;CYh7z=os!cU}4phc;E zxk#IVky7Ipkv4xEeqXE~TvpMOs7~P&%+%q=S7HP-;CO(!%X5 zptR+?A}xY0rLA8V>G^lEfYL>%7o`JV73n6(fYQQaB1OPJ>7s9l^n!K3gLLSAk*?^n zfKv2fk@mln1(YrWZAx1pYf2aH5o!Oa_#SEg(;_|ZLKaZk2b`3eZx(6eEi9mP`6oo0 zMcY!Ue^I38eH7)8<}VQGM)0Dv@hc)d@iw#{(y`Bp^y=@S|0B(SFQx4#igf&D7En52 zw@4@LV*#a;7e#u~+tHtp&NxM+YtcU_?S89B*MT3UJqJa4>AftVbnAAJPWW%oMS9#L zBApDmQaa=PBAs>#^eEEtpA_jN(4utxNTj_Fvw+eb(4e$$n@HE9E|jj@B+>L%O zmeRKKM7rv97Ernce^a`Br$|o&eM+OxigfY2SwQJh;HEV7N|9c4CJQKC^&yeA0}G`a zP8I2Dhk zNW1VIr41;{6Xd!9Y5E?K=1^x!r`{{li&t4d>2hGc{E{%-aqacj?b#P@yK?Updv;vE z@47wVXnvlB;qDu+xhDT5yBpt%@AhrKardrp+uXi6 z)An*dw0)|g@BR=_!o_jPn7Iu0Ihr_7TZbdC)xqKd-!b-QOd^U$(x;+fr-EcH$c8AS{ zQM=bQBFN{e@4jkpIM)H|<|t~kyQ65(ZFU;yVvWX4bJt%vcQD*FML!xh#}s`^@Vr+? zqifJ9_qOlMGYEGbOwldO%PC$pi1MTk=21HH)thmx!_{^(Txxb(ooEoQZmA!(^bB1UK`&c4|Rq47Gcu(ufY2fd@mGOc&Rm0Rb4WdEo+JlxpO z2zMz!lGbF18ZZiAW25pTW5Utvj@sRoCNy32E<_uJ-rGBOFzGWh4b@}DAyHo>ohAER=+l1{m<<}3iP6g-b9c8g zSiC4?xtHl(r9f05l)>WYU1boIFJ70^Za9oa;UHQJ7kVq*5$~#^M{;GlUG>EB1r+U8 zv@nR4qpqDpxKo=LVoM+_S&ZhL-jOb5ocUwn!pdMkj7Cd?sM*r$LFza&{dg3rcYC8K zlyi_q_C}g(u2$kH%vT>qokfj+&^fi~h7y1yM}63D9_fa#7{VjX_9$HJ4ML1;jou9e zNa~QV(;ki*)7sdrTpn6%vANP2h253qSh_>bHXJT2MXeRGU}%DtdThI@Plb}`JD6mf z9h&*M+B7lYRo$*nte&MjxTqE`&jV%ZFlv|ET{3Xp)lky%D4|B!JQ?uq?xC0F6NHK8La}t>2&{Bx99leYocaZim zAR333LX_#;;{Y0il)}>-jj-z>M)o*hs?t$NqVCI9qLp|6mV>fYO%sioM!3_U4kklY zLwPh>SdpsG3uP;Cuu!+mWEZ>>rjt5(jH-B zGFIzSMT!*|Exh6x;3hE|(*3M9jdi;|FeXScwJH>|P-42l8dRfEU-}vI*;cqzaheWC z(NXhq*K3D@uPG{#G_7q)5lYNhY@aB3q*tHiVP&Y*vFF@96orQdy_G&6+!^6=dyRi9 zg_ts=s}`r#il8dg743{HG`kB?rxS_wO8j-)auQW;^)d#Lx?q^aAyizt%6vDLI;OsM zs#=`(@}gLiTG(icvs`A>r9WC3MB!qe*09)aIux%`=>-+NaYZg zX;?q?qi&0g_~__D)Tcc&XN#mnRhdG)UNiLcq}jD)X>mrZZAsI#djo9Ly0IUak@>>R z0(A@$bxG5~d+S_%F2h_+o0ZGgyUpdO5%;H--hgzPaU=E~u)f%)7?sUlKeE&udC)DJ z9j$H$J2#hV^`eaRF(z6y$VSl%4vj6*QlTsl}$$yi+=V zeW4RI2X5M`CzQ)v$9`7t2&~^$*l7+&wCB@XP~9wCol$$r@d279lWK)yo{2ItGC44# zzHhf0Da%E$Ro|OH8q14bpRTN!2lFF|_xOUWg5Gj_I3(>V>~QnA6`ND7 za2Y#d?S3bcn_!^t1nN^R=Cq7soIWN8+Tn52cIFh+a4?4r!};DI-lwz3(~+&-9D)Wv z&{=G@JD7O|C>YQ*V75+v)`B;^-Cg z?qFbGa*#HKwkZuzGB=+ERn_C;T`8sOH)(>RKc=|I6c?I9E0nGeuy5a9#(JkY=u$UK zmO4@&P-SjTj=aler;~n}my@JLQ@9!phOkfKb9r~3dV4fN?}h5>_c#T~Qadn1 zVr})7n^@mR*ioSAha?&9s!bo5!J#Xu`*yb#9ZgS6RBUY+2<*|Q*=mlO=rT~YE!>3g z1D}VU)&|>8gZ!C!`XhjkGq21*mB#nwIT1<8NaL7r_u#zb)>HlEnj1eiCtpsYe=g# zUCQ_+%A|SG`MS=_@^zgQRm?3-AlX!nzty)^{R+kFIs^IaYR;SmJ4bg!3%HulI~;Wz z={)?Fg*>%S?KAonA}cUMnhQWbBzI-8mzUr&yOX*&t%O}=v1ij;h=#aBI*OJX;@U?dnS8Pv~hz?LAr!6I@XU!`76nfOQud&WXB~UEQ6o{e9yNnX8ce#3O+c$;_Sh`}% zf|jlhR@6V}s71SyFb0Ks3m1D14Vo<&kUbb8_+?NMiN47_CKc@J^5sQI&f0E`f5G}n zKdJ8#?7dd1?^rZ=m-lP4aJQk^X%LP>hU1{_#@oFi8b1^=5#2CZ!^Sr*sjutJp4^n@ z_s4t)B)rFA>s7WcuH~04qxCWWf^~1HYV0bNXo{CPr9cl@+am)G)n`NlxK1R<*Q7;vLIZ zkd>(`)Q0N&Q7tD@Z<-2Wy!6Me9c^$9`&j6w4VNouZ>4J(^Y+La@NLG?(7xFBg z=PRwVihN5JS9~>ls&fCWLc@%})e&6{ts^K)Lm|+#u`YJnz~)QK;)y)fO>*TFCu4_9 z<<~9EWWFw0O!2XdvJ@xH`brp;EvGoSR7fE^uflnrrp}(?X%wbpTCTL>>KfW9t|q~H z70&Z4#OmT%s<7f&G?r3zHA>i@f|+|E@Ivn8C@AikzHxN(rf@J?RyKqjC}Ja*4mvk$ zDuPTh-O|MEP~4l=_AF#^xq_7%cVc|jH5=A&vNm7iY~#O6V+H4IkY zr0FZU- z6+Ojv09P#B;aZLKgP-E1kh7TLBf&VcXN`?gg%l4{Z?^Lz>kVg2hH+JujzVJVc~&%4 z`d}$y3X^HnSxK0Bs5b${TlF$|=%@AH@`sCtLe7)LZ2nx6n_Ak2P!+D5Bw;J26q#qK z!V|Pi?F?PW)(tIKbqc1oTfj+T8&?s-A#N?ub)MY)?c$~^PeUPqTg?%i#78YKhqLHX zbGU@tdgR@6h{uqipg)oitvlYe(Vx&R!=7L$kZs~&{BwsO-++{a51#9>FTDZkY$|f z%TEgC;8bIhv)VW2iIq=r<;|LMM0gwu5v#|xJhaSiwAYtjJ9Bl65k0c5G>b{8w#u)> zSFxpL-`z$pnkGB^zELgnl^V)z9ZZ6Ys)cF&=HdWNrG;2eU{)lf z;Sk|9lzMQ1z)+MTq(y-7x<*dl_!88x_4=vxZ2=`Y1XMf6?Q3XWxYA6Dw8G2xN9R50 zhPx;zKaXL+J>(5QCqXlV5Pm#O#1O#6-_*WOfpp6uoe*t>m~4IQ+rs5ZGYQ;K$@~g; zK@I6^;bg>&O;BN@y)z2bS{fO>!Jr31kW+nSLh03)CKmI=r+N6LHg!uCRP}ahU^C}6 zP{QLdQgta%pe82ORI!7ShQ^_dO$6X=l}u{X+YJiDrH@OJ)bQEWrkR$Z-76{O3e**9 zw+=OvPN`;sjB%?+UXVj~lC(33i*>0PI)CXUo>YpEPnd_QL6Mp@Lmy9dm!YW;x|Ob( zRy76grlk-q^p>%yZ=00ORD{In|71th&`bHxxZO?qU@e%aIt-6AQ^!(;QwuaicD<0y z)ngG;m>kQo0(e;78I6eXDJpd6hncgJ0#LLdh8;motl5tA>om2$q0QHTg~HD@pr%Je zKq|0AXaS39WbVpdDt9dzCNY4qdD0tFIMm!Zl6+E22|IZbrc4>cVHCm6@d6zB61?j3 zEHd{&JWB8w!7l8X9ZCuUTqZjrybW52h=$B^y-4o@IO?!h52IbNwCZWY5niiu@`Enn zif{)h#Kz4gTe}jG`W*7_@ieR>Nm znG)B6hpIsywg~Vk*(+b3IN-xg2={HR=pIOPxr z9K=g`>PT;_mNdwEY#wE5Krb9gNgLXe^p2@H)l4S~o0=)EL3JLg2Jr=E1D6TGpEv{D zrybE9aHuc59&)JDn-}4M@LpXkavJqpNWV53S77O`J zf`HxihJYP>FkqYC7_dk2`)Ba``|}q=-#>Cqz)nQ|&+ZJ^2f~1zj?X`~4g9VQ*kw!L zzZ$U5ofEJVP=0Vnz}|aSz#2~t*hj$YyJrUM3&`{Nrvvw$D1*Q60{zR51?<1pf#2N$ zJN0P59$Nvv`vdl+69cw=6tMN6@m=7$_Zj%?-heH=8g;_=r#%p``(BUVksieFkD>lw zIuCMq4tT#3*zW>X$nM>T1NP;9z;=S~nfQG-e&5sy*t?dIdI9Uc46+#nY+o0+@%N+n zZUnh5g8v!7bsy4cPYT#^knPDAfDdHb0qu7K>(2Y|IdHc&1?)Rd1YOj9Kk7DmUBDiC z4eI^cfL)06P2~I7Fkn+i-$CBX>!7(gV7D{^_Hq3DK7Ou;Y(Mn^AWzZG>@1l9v+ z!-D~PpcSy&7a)f%$cxXvj_>b$K6qV(_D7!MZUQcRzdl0$!Ds70=l(;e+Z6%(Fg|3ST+Xi6PKgDvjO`e@O>2DU4T4y-Wag2;Je#U zzt0_mTwjf{Gm!0e^v~mg@dU{3B*^RJfV~}ZI^_(=<688g-6(e*>aquI_foXwtpWRQ z2)@N4`tYM?VP~UqSiN_2`Rxkq7y=0n_(DV-w21`2u{8vgd6>f4d4lZ-Lxz zNB*awACAxt7lZbt0ej^X+V@2P`w;Rx3~Z;~5U{sEE)P5&7+#1zhdk%ifE{@bUJW_y zKwjjzDj0 z{@#V}HvlWrJ=5Slhx(q1G5TWAhCIK8q4h!bJM5R)$JlSOh+WAZVV_{X!amOaiY>B3 ztif9BRCXGB68i=AGWMTriG7IuHhUU7gFT&{$!6IPep8YlZGxlZnRrVk3-`RKB2iSYq`&gg7oc#=Y1-qNQ zgFV20f_;Jg4g25hBKB)4yw zU$Vbv|HwYd{+OM^euI66eT)4Adm;NQ`x<*U`#JVg?DgzN*!}D+>}~9|><#S4SdCrA zp32svpRb} z8?qO()7fFxVK=h#*mky$_1GMH4tpN^Hv3oh9k!G0V!y~%*b%nMy6h+$v1`~ib{*Tz zu4UJ={S0G{oxo0Fk7FmZ6WQb06WKcU1a`{*Z|VQ%TH4gg$1~e4MrHePD@t8qv=zv& z@Z4pJoD&+>+r#nIVH1!PQ41m~UCJ+RC9f4}zQ+mT=`gCar9mf#bjT>41v0$FqO!;> zp;2h{?n$n4rcNJK_Pk_VB~n5)^@|vL7FeoWo+a-dxuS)8vazJ}xS}%LE(DmCtoDT# zl^qL0dR|&*;S^uWe^!LY;&z%9Si_?q!02 zY{g^)8G)4HE7;?)WtZC6qe9SH#L_*QeWTFy;#{(It)3`%KjTf}rap*Q;e-~DFf;;X zp5QJ)$MKMM+0vz6y0a20kT*U-Dttg3We}*khx=)c*_`VnnUO07Zf7N`IW|P}92vJ- zK6z-{+j~<_<5D;DIm$|+_EZZ$?eiFG=4nkIH7pKmgc_w9rk~YxYv6*iWt6X*ghTI^ zC|FA6n&`N+ZPNYM{<*2DhnCppEc4u?I*$?0Bn}srqSi_W4}*#F^VsU;gNOEdIn6fS zI)jbH4L^HsF&BBhS0L@Sah)hkp&hh{37BBTm=+$=YD3sm&K@R%=IJsNi~)rlh0Dex z&EYl{r0I~WKwfJy`IMC7N4KzjPEqkaLrzf*^teeX0$!ejDK=knjG#r!EmtyCB_0>h zT5c`sw8l5<*X(wxCZvi_)hi0~6jz$P|I$pS&U$$2S3hu4uEW!BF zYH0!{D=<&Paa}csK-|pWnxB+gwmuG0CM<@VuUg&K_KutVD7;#>pwABRtoS%bl97O@D_C4Rcp36HdNL!p_=AMR^?bY(Dr z$6Yw=^suRj#Yu>(d)=mOxi%{did}QtQ=)1`xLm+PMGc}wj1;<#rPhVi4R|yU#a1p2 zOWq2|7u0$T?`0V{Vhvli>styt!u90=Y2%~x&@l4V2>A^(6Sc`*jL~W=AA2ec=3iLG zF@}>hy{E}6VytOGq)T!ctv0ULbkya?5#`6r4m?M%mcNK$y)!jI!aZW+j*+7`rWyqfLft#SnHGD%!1a z%hhy4mFc#N;V2%{A&1zgn-vlikNC*Hz;~kYqkrRMtW}Ref`2xp|b} zB>SsAb0OVaRrwoPbu)GlHoY6V4np}JygboMS9tDf~EW74^W6r5_38!9#kw^D7#p>Q>b=vgQXL}Gm>`Eu*zNO(}F_3tq0vjLEP%?A;PQEm9B7Qh{w7O z?uAJ{?6n}r{IO6*MHilp+U{Bsi5+Bn9oh?^(|ByEavh}9FJBMPo_)NBCr%pnr*{y@ zp)lT@7$N*m500RHc(p>inNX4eo0!$v{QOQpeArv2@N)Aoo)1^DTb)(dxg7$)2WDt_ z*Xk`dQNIY+TIkNJFr#bJ2WGJAN5^IDZVT_KOIrs8mf=X4bjWg}dz)x3!3 zNn`&N?WofRb3xk=cb+sPx=m6aM2Dzz&Ep4N1K8pP+9i>gF1=vyNmU_3Un6W>p1yUC zSLvJrwG{q~t>^7H;KJ{c+Qm+dSBt;O|ElT<+?|0HdH4H6Yo?N7P^YIQB zi`TCbF7a}zT)1Uv5}xw@;NA~zQ-RXE$;lCnSm-$}g2ihJ(rJ7($Ms$HP=FQu-0Rd$ z9GBI$&e3opfZZMr9iWlvY{GqcZimPYP1;qCR(EMrMSw_cXsfj!V7^(Qp{zPUmtXc5|OcrCOnW ztDIO-aS>r793AUNG{&f6H1vn0(EI41q%M`c$!lVqx#B&%cvz(#)gp79$Y}Op(t%w0 zJE-9XE8T|Sa;DgYv3^O*aD#xv ziW;RxZRjfmzUV?2c|A?N@MiEHMx8~iH|?DxvT@5$#ckIvxqOHS*Ic9|)rI2t{;?1w z?i};5PFuHar!aE##}xmKYEI=yE#b0wyW3wGHS|7{Tvw-DvfsH|BGW%Pi+=M6CIv7d z+bUc{%uTxRpau}HNg~V+`l5*3P|qx()QHMllA}%>cl;_rDrDKUjSWFuu$G&L0o6ea zMUnx`y=186U-p4@6IV=M0vhujm{D{Pi37cd1h1_#%8bv{GO4I+eyk?Ng+3z!K9iEL zZ)GX3nMoBLTNSQ!eK7O(iI*aD(O53WaHJ0tol}y!&?GNv-6KoQQ6t?aU~8>-K~| zfzNl}c+EBWFYyHtTyElubQfWTmixma{hIO9>CC5#(P&`_25L69{M?v-Nam!m1pg*m z28%h^=A%KgJEZswyn;I3eiM?);7U4vgl36anQuk643qEB#G%QnkLJ0|OhY=n%+px; z1zDMrMSdiOZjbT%gKPBW2(;kZGT;_M!Xj@%kbNb;E_8=f8Re;C38zSRp?s@K*GPN0 z!ZuCSDGN1KI?tYWDhq*vjY#1$n&8$A73mq7e3@>E+}8%HjcYx!MD!*zIZ?(uJX|TY z5>AIoEnlHqWt`V^bzV+6S@WALtn}k#;v%}yHptIc{I&{h25GOYR|!8Okz>;(Zo+h9 zCDvg4tscay+#@#z4M;O}!nn-)a^{|^lbNUkro@&b>UIK>MzeIow38|04^*0(!!^i{ zMkz0$0+KF=A+ZstW&v(<(}rkSCrv5P60VvmtI=7~yaeSdEP0<(!7u94Zk|M219-xn~D;zf=#Gm_|hVv+eGR z*toZa3n)X#F(;HRUTQk9Q*+G(e2-r#*y;|XaXW||Htd)V2E9Q>6Jz6MJFUnSyNWSv zV_ssnOPzzI&Jg(wqLhrfN-6f~tb1l!A1z--U+(xhnI=`io*sp$n9QSJ1_M^JdSN)| z4yQQ?hSJts2)x&&tmou5rO5_2T|+1)VZ2Eo_G7M~6E1mO%Tg0#ahlrs+J;L*?z22yU4k(W<;m4Vonnp3Jz`BWwOc3?qizk36`<#HG9^o5Y>3^~h9Th(UtM9TP6 z&OGBjGgV;B`7x^KvYHitMaR`FnTqbKo@7h{jGb6>lpOx2rq-4Y0i;Nc{bvI2uH-PT zgqSFq zZJu>dF0sbTwvHu(rDUOYQ04rzgZ<)gwjrvy;^wGK#346Fb+T@`S@PpJ=VobPUlW&2 zevq$$tik;Tl z={-M}-Ymk2JN9N7RB`VuO~cZ&=%`C=I40KIxURkiSRtH8F;It&XSwtKNED081`%)d z>LXMfby=>mU`pdoyE$A|B35}>>EKY}2lsHW86JtM;4$Jt7hbjM=KFuN+eB?(uXZ6t zGgddzs;Zl7io>1h^Pu=Zp_rHrw4I*Q={PGef7j#k6im=#2jm=)DsIR*BI7tCXNhI* z(~o22iOcNCt#kAIZ5Ebk)5$@uptDQ&ar+IG%BI+n-I(IN=S!e<35Y4md1j`Rg=Zmsy@}+oaG1^ z%h_2GDdY02a(g&FCxs^G{wzsmoth2)q-4Bj?CfkQ4_9c7@!B{<>kQV$E!xBZ+tbw? zg;9|FP!gAJNLssTBX$NR?jo(#U@S-J6rG8?OKa502rKV2trx83I;~OQm+L3#K%J%S z>qZ@yFYQbn=lAsl(~6ZoImn!ldbn4|+$P~LUR}Hb80JtLb)0Vu7wniq2|w4Z+>=^nl#bX*`SIMbV=`knWhWGHIR*Y;`T<$-;eNb+ zHkOiS$ig_x^j+E!EOygQuvK=}mJ}Q=+eKOE6yM+iq+kA)&Wc6u+c7JL6L*rMq${_| zU&WzYD_!W;t>u+{+1I&SlBnw99hc86Os$tz)lS$L?%tB;5}jBh%yks?NvFvzbJ4kZ zEF9bG^Lj^#fgEc`cyi4|CWX0z1Gs`)%?(_Zx2%UP8o{sx)z*9^wL?Ngzv@P0e5O`? z{VB_O&97Mk<2s4Q0`+$lmj(NW26*|Un!|WZqm0|Qp4INckvNa*+1)PWshl2;0G)#{#V;LTid*Gc4l^i{6zem zivOIMosECHeO>FmGmhgu#& zW~neUNL~yi0n~{t*hk*tq+@;&??Zy|l}fE9m3dk!w*t#?7A!@M$%S!1Bv^0B)oNRk zQK7ft0?-~`2CYSe%Z5A4L_x?}04jg}ev9MtwnoVO`>kgRI_v-H@Toi>j>GR{sTZXV%?s$$q= z!SDiLc(TOc3~*GLvs@}k;gtVT2^J8E(t!~Go^ zJV!_Aii0B@3pC~G+k(j^RHg-KJ@V`nin1}*3)Li=MIi=BArXB-8-#(?z zG18iVHcQVZd_Dp2D!1WgOV6GIisuRx-G!)=eR0Nlc+XEkt0S`XcCzm-5QzN4^i@aW zkk9itnu?MCDkIut8FiOJJ{NLCB2bLwGk*6TwS{zFCeb| zArC7{z_mMdk-)OpM6`2qN==@0b-((tU0;O6z{;V9dKvu`*`3Cdd@uKk=T zfvR+bO9{ePGFvSBIolzb8pkkR%*?8|T(X%aTpm$4t7524N#UgJfIX1S296;Whtp>j zDz4Mg>p?s!XH|r*U7(pfLihS}LYTH#=4HAuvC)93%*xXO&3nYTY^qIpY>rjuq!77m z;aLl!QY%giKwtgfnl&~{e6F=7dFfJ20@=m=+stCs&!GF-gtz|rmq8U2iqFvulUv=F(N6(v8#k0w=H~(V7Q!09GRqnHn zJ&~74?rl7Eb7xc{9&Tk678z_tpSau)Me2w?##7qnbnScB1&%uqMG!Yn4@cPt&#+A; zg5`1xjNfpU;_x1~uDy|lz#)Ur^4Efro~plhVp-&CUDX?xYwXOWaT#I*XJbOp#$Ng; z$G5$5m8s?p!gshr-#?3r7B9>t#?2|SGn-``5E)Z)cuO^O7P5>Cx{YwX5Vrv zkY8;?Zt@wIZL51^%f^j=me!^&ZLz&HqEy?aHsO5nxY~Ay3(pU|ZEMp~(WQIk654iW z8u^-p&#u}7YSYpHP2)Zc+`G!-w#;O5yV@C$TSE8Nm9Dwvnl$!`F}@`<-+t^mJ-s%C ztYXw~3Cv$+j+61-DP*Q#IofTm+=+c0KD;_Ii*~#QJ@GmpWTw8D0q5JEwJ9mK^e7>x zj_cFd#*LOP?m+rKA zw`>u(_hxWsBjJYK-l)BJj6VsStV{VXWy8cv*xvotfqGv_)YSveS4N#2^`!gA>mlnK zi%|a{LfOlT`DaL8j(92B z>J|$UeKl8BT4D!;zgI8@R%zO#vEB}>w-#e9?FpXjZ^h3yNWk&W(yKz(>C|mDyp|^$ zDr0br#n6R84#d==RKK$6D~&A{B)idiw_A{SE=R}E6fRG;W(J-0HHn-pU+|QKY*cUk zQLEWU!hBh@W)l_rHTJhJBw*jRckPcXr13(bj0&%PXE(om@wob8D~)H2_c^Lc%QZVh zDRd=Y{E$Gr`5iX}0Qq-s-Uy}2I<4;LZyvmZ2=zS_y zMMihnmeZK!RiTD4K&lq6TY~YGpX=Uvw+xnv**vvrDVDUwO3^8L79@T)k3B4HRq46z zdi7J7t3AI{6<2La8jxvO9)v7kmc})a2Yo8zteH;&n!q_oJ<+hiRs-ktZ_-Y<)#kZEKqN_P% zc_fc+La#SfM(fxcK3WB>xeHj{gQ|>rgS;S-)}}-B*JDo96Mtb!R$-!g>&{di#(A_# z!i#^F*ISN$@p22g$s2OMs9h`dyV9r;s9Z#VMU3q=v*ld2@GDYyCjL~HY%Oou%Jh4wXeSjeB?3b%Xw%EV9BU#SEKp{ECl$z1UZFt~23R zc_TE7F$dDW-^t~7rr(jD>Ah)`)+f*m(vuGq$d2caOPWR*Ty%Hw)g^J6WtN5J->&9U zbh?l5njEsR9($`SJ(vCaS}%-d845RVY|M>ypSU(98#!%H?1Lol%cCrOKhDq(X@Kw- z>7*$ATid)cqnHt|tANog&m2tu@nZ2qyAC+-&*QB6Y@QUOH@;fUe23!};CsCpA8Bpe z-Ut_0Ivv{sP@C7H&0Txx52Ub~qS;S$?^t%ej20vhl{XY3vESgA^xw#=F}Z9Pjbz*g1Kl5slwonfWU6IJdA|=OYi95dTnbnZKztBGPPVy#``&j_J+o zlF*xtcwBe24V%OMeO^xanLbZ3x_+cIqRC0f0SFoKk9+a zJX2=jdFu;DY}B`A(M|eY%<3|A%y^pilba108;*9A-ct&nY3*Wy=G&gGIG*pdA#<(@_`cx&bQpPGRIb_T zXS`6+7yx&%G8oH8(Iu&et(aV^gSD2_&w8PnoTO^gayX{<)|S!Rf366V-x9g_I*GR~ zaYg?6`4oEjkg@Szkg0L~u~=qQdd1^?#aQf3>W!pUGv8*J2a;bfBAIkU`sxtZy`cA} z5LbIJI>y7iaamRabn5Yo9M@P+P)k_eE$oW^@_`svnOChP0AIPdZiW6*%-`dkYzf0# z4z9Da4+87;$*K z4_;Y1u2cA5k*L<@$?@u#9d|E3lwe+)SIi~o_~%6KjQna2Gj7~Z^j-AIs2$!YAC{=& zx6tFqzPIGY&cOpS^dNhyx7=)Z!w8`+>FFeKI9Z!MFoVjb-g4KLH@;GK#XI=53|{_# zxF6avkh%PNKBACK{5ANhE$_1JeqG}AdtY8c@t$W~Yn+cJSgJi)FG2bCG}leW-;kL2 zyEE~)nEhdni_MlfkmBlmEQOYjin3V87!R|Sn^g3;=Wi0CJ9_Odtw*g9>#~81eOu6V ze*AGl;PXE5Si4!Sv)bR1NJ{&tj4c<-$uYe$^xK3++)A(nv8LyopxS9yL)+lDV|ai&`b{Gb>SHX2&j_0~r&-S~+FoxzIa+|zv;a+(?k ziz`0o?-G`7b2)0*LpmofKgUhP{G}nU8)ximdL!?X96_%Y4bfODXexKCZ9N2~Hx5^H z>E9DLMp4*YBu|IYF&+p~>~~R@J$SX&2)@15H9vidIQQno@h+ba)Vuca$AkRtGatkJ zy>!_u-k2O3|Mv+~f6(p?q#)u)1o(hDIuc^J0h{#oxdSsYV>oJzH>NG8^Yr4-rv;+k zLNpwPc!OfOFEAotO+enjS^rfj@>j;sCtfRK@eJH$y$ z*vIrpwrX^AA?lCXJx+L zehX*E7`i731e9qXdh_A49MeiR$g5B;`q7rZBrG-p-7fnRCs$N>zhTUzLe2X2QP&*! zxfso;y&Q#zh}?jl(Tn^q3J=3-F`yxlxZyxD} zO-O|{_rt{=cAjvPm&T~3wv%Do!9)lz0!35dgZjper_33!y+2zIaR~n{aCyM$eNIbp^S8aG;6V}?O zy>K`-yHgG3-G^L}IliPJJJyamMq!P&I=0x2bgT2PWA3I9aUnzqr zZ#yQiQVS#h_H)=Se=Cvs?EQ>&S7Nher-OV4{!T~Zwf~cXyZ~XyA(=g>NKCmfHxnjvWng zv%?|;|2|;3x$%7P{^x}E3Uf%kF<^J$Uzb3iJpA+HssD<0K zdjQOAGFacpA`wi}KH&F-+dK-*Wc&&Z#p`_7DIG@b>%-_7yuH=z+E5Bak+ nFW1Qb_Z+VgOkz^}VFW!KwInzBvpiSF&d-13xC+xG%k%#Nse*6D diff --git a/sdk/libraries/x64/tbb12.lib b/sdk/libraries/x64/tbb12.lib new file mode 100644 index 0000000000000000000000000000000000000000..ace282a73b612c7040d013e75eb3f194d40ee3c0 GIT binary patch literal 47260 zcmdsAX^><^wLT2{h&loyir}CK1Ip0V(=**Yz*KcJBeT&A!)$Ecsjj-+Q&L^k)WQs? zfFL8Ah=9l<$S$&ph(P$mF#M~5M7%#nJR_bz48kD%=?{PSf_F~l&CI)HR_498I^KH| zai(rp-8}i_$?e>; zo9L-mIsK##K=hqc19~1mk!~O5bnA_X1L02KFFq*(mq&M+Q-3gRSswZ-X&I{=6tM7a>Xbqc4~Rv=%b zyOwc!^icq!`~Jx3u@RI9($9OG?!Y!NLVR~1JrmHQSf1$h{hS^^IS~EoFsIw_ljsS& zhv)}$obE>~L=T?G>49Ug9MVG@IlX{wAbPmN=`fZfdgyPQUcmc^zI!{TJI@0kdU1x+ zo$uq{NWZv;)6E|O5dG|FPH$p=6WtG-9!LB{KgRYF9sWC~`>@?aKf=BwdiY65a9vr+Zi8ctU#Xx14UC#(I&Sy_wUmZv!BD?rcuaPhdYHJ@-1NcizVFg!Jxe zBG21$*^bM1?(#Nny7H2pTXyWae5co&oh^-m=gsWizCHOf+{%9EzwX++duE%racb8T zl&jsY-|02lEpM*cX!bjPrBkZZ{T}{N!6KE)Ra5(RY?|JjN!GJtU8pMhV}wEX2}WVGMweoI6ynlU zulA}Q4kfQzcWEmhhUP&?OI;Mh}A zKTq4pQp}|e&evg96>g8)bl_twE_8-ABctf3UmVJ)5A6(W?G;0Qc(vi4SUcc*)n;S9 z<=64E*{%wg^x(kOP+7t}xYBpNF*KORK(j0G_QYUy*2Hjj1 zc)bOGq1{Q*9^xXhFHlyE(sGXp#i2yG(Iu^Pf#LD`T|Z^9_GXkkY;TtmNgpj!o2F1A zd%9Gw*Pi``U+Z~`?RGOzM!aoPo2GaBhx}Tft+lk7RMtlH4nmf?m7ze6 zWi2u``K*9>Giods#jcDxqF?Q%>|K0O2ni>SQGv@d_ZR0oh%!EhubSd>p&RSkm~>U|=H^qbw-2_8!BPOH@H(L#8fv+S ziN$`!D2{gov2K7Afw(ZEk=XmjcDw~%|2n{rY8o{K)hbZ7k8m`61ShZYRy-6 z%snxMfK0-}t3>t`GtA-Bcsg+UHGr7x7?+~u?qc;|nhr~gT&NcDC$|K9wKfXsP%rir zRS2&F5M8Q9PJT$ltjr%C(R$IUr>d>!G7%CAsgQa1)h(9%Rm05{bJyd7lQBHdR2qYPik5be%5>_YMp#i_4p zm8s%FRAswKcugA)bGd$qv6Z^+rHGNsIJuZ)=2LF88odUZ`_hbJa<;_Qllb_`#RIJd z=rU-h*w1XM&(44>VSMJRDN?KAe$;HH8^y%(F-RZf#UA>_y}3mXh0{Y@MVf+Q*<-~pD=$!s+2NI;A)bZXbtD5gdxC`nu`yv1q=+>Iiq}Kxl!tsVJ_>_e31c4h|=r z(i2q_U$PqLT%a^(rXv$$pd1V0R-SLRXRA$ITw7>PX?oRKuW=)q%&W~b-PhX|+IcdR z4C0$L6EVs#@qEi;qaDi*uoDfa(rhEr6((PW1}4+K)egaf=Myc>Tpsn6mc}XZYg2`z%A2i0OMwQG9X>!9VJu|SDIxW1F?ojmqk zbV~*?59WKqzv|10UovE{PKpfH24H?=A9L{z0{r6;z`t+BJi(U%Ry~6GeVBWA=zEyg z_XOq$J`J$-8N3-k&wdiGe~S0x=c|~nS^o*Zso%joKcrDCb0hM2>t2ArJcfVYg878s z258;`@ZQ}3|HM4NGcf;efa#Lx9_;P2lAs9;Xz2j2piK&re6F#CPXm;5EbU-9=p z<8Qxz4d6J$_0(N>{XD>L5Z?ym_ueZ2EzGmr@k7M&Aiyba0j$7#m*IVs@A=33ur9>W z`#IKi2l6Arb}Yl+_9K>;9>6@vUm@Lw-(frFet>-L2RIRV9fQAZdqjlGq4vff#cy*@NxJgtbk9z39uYK1{XsO ze3*whI3F&9EwBT=4!husa4xKeufWl89gIK;*1~yEhR?%JSP84(9QZQqhA9|>D%9b0 z_yU{(2cQ8*z_R~W?Ku)NP${mssJ*Y~C-(vT;t{ZV6M$f(ed1($67h=#ty!?tJtSsf zX-vdHETmHDQ!7yWkh*Acd!XH6APch2DSLf5J414g4HmPOt4a;bpK;A=&g!{ZiX0}( zR3yC3HaEFICw)Oa^@!m&w`n@UEvMBthpAM#0Ca8?_Sl`xFyfjnox~ID!DLo zh-DyT=ovM+7O}Naw~*07#Z>aVEDVcT<|p7#LEVZ*kD{dQXS8JvD8~9A%>cq`%pcib zyoGkyzHoc~w0&wE>IJ!(i;Yha+=d+Mn$yicxVR~L6-yW3OV8l>%c8o4vFH#`>+I^H+(?B4T=CNkhJTXGQk|L*t!JoblCY}&51f1v z;2g?PCXBc9nK5j`9oo4h3=^v&J+Dl6DiUc+c_VOz9XI0j$>^aVt9;VXo1!@@?O0SV zP1T;!z?jH4e%yQZ0Yv&JaC{7^rWXq^*w&a9|R*bJ0+-hmc~`;taAS z8(__uvL^GNLzl}(T|g=uNv)t9wjm`P6_FH+b3I#Ve~!aAh7zG>!!$m#Qq8^8@1Zz1 z&m6UsGTp4W>deSI5X6OR{sRJv4*JTJ*I+2boWm3XMR~70{DWkiVj2k|`SPCjG(TUxi0E!?~#9MAIUjP=crK=ow~v)a=WtGc_FVY7Qc`T24~f zX_pjtA5ygS+6cz%jB~rF1j2lb^tRm?e(J^ z`0A6B(ZHot@wZt_P0^;h^;Edbq-0Eo1q>Kl<{HC%YQ7VmVhr$-%|v!AijRSXo@cB%K~=3=j{sNj)A!Lr!CDWhas@>7g=) z(_U~!Cj6A~QL3VJG+DJKr<>@OB3y3)FZT?}a6vmaoGejH)-VDQoPnt3_Ufe>;eCf9Q1>&td3zSfVR7{?}WDqc0C zLuj?Ggsi-CyD5X-F4$Qaa5Z=znO>8yFesrJiaI+kLug{Qw~TQ{)_K?Ya&HCh%Z#ILZ)=BqNiYb`%Z*|?D?a!ZeCI1UBGZAxH#UH_E1y)S2Zcv8SS)<~olkwsYNdFalaU_0L zVB~kfed~g+F)tF}zmdd!^nC?Z{%#q5uD~2h`ggo+)H!g{ievGc_u2P=nUyQxmBxve zHENx9w>{TexoYRRE4Oc%*%tgB`8WapA{az~sh@PU0{=gnUCBD2Qfn_PBF_d2mf)T-x(5qk+Gg<0TJ;Tme+K;*{3GswNu=uEGE7PZ zbbJ-{qR-{Bp_U_LbZjCtLYA=+f=RVZ5Ck)Agw*MW(7(_KStGut5%Qe)Blx5dGK8MD z>Cw{>g7&4U&N2($aJ+J*npOXa74Ys&mziN!trcRTSxK`h{I3EPF{{SJ_jKQXh(Cf) zy02i_+Q{`ZtiGVNsx+cTBH%d0%=yE#UhQKoJgVRfV%g^)uPkRwsU+TYO>ekXIIl*W zN8rLEQh<|MSGIrr}Y^FGJZL&CuoxjX_N`k1R*i=Mn8L#}t50?`>9~ z<(1{!pBPJ*XazAJ8^#>dq0PDL5`_|{C7y9Hd15_GTgq_}JUWNCH1sME&U1KNGtMRv z#qkV`)3-FEN4ivg+eJv^2)M@Cd43*_0UC#=wpN^8YYE_6@%W8DiJu3 zmWfijbS_3LCnT|`XpEGtjK)pA`M+*59Zd<*d|aU!6bGin;XZB+v*!~Wfo*VE4iu#k z^^6u?rFWwoKFLuO6|t5Jr5HTXz2pi+xq_oCA`C7ULcRX@DTc7f*g6GNURkaPrv5%L zjx%4po`eR@b7Mj?J2zo}pQKP`2?ET4YFx9Ll40Uxj>;;sP(hGamc3`v0O-W2YzKBo z6wYhDQ^$EX{y2gN}lr8&@*3{MWH6@YFiPYxO#`4a5m z&qp!aJUgh^*Bo*j*F4RTmGkr%Cf)Oc2&h#svji`I`gPq-Lcj1M*6SD*u^03ZA^O?8Q=vpP5NE(XbYKEuuR!g083#Wr@&v&n%6K zttyK38;JFbgmuVa&)JbrYP7P;5&4&#kQ<`16+GvCqcoy_N;r$-DOA0w%9)``n__JjKKIak z1;%-;F=;Pfk!bAV{S^#(W!Yz@r=G8J6ghJUBqI0z5=&idoNvT&em-H#oR^RTlh}G= zn8-rAI>6){UWd63*%6gSg#sE~X&Z>>0)c3F`4bAb^9+z~=$I@wAo3B0+&w8n!I4*% zGr?prDiIpHJdLz?CkZ9PE@W8iN9|E*vPTp!d1WcBQYl?ix(54h4Z~INOMfLmcNvZg zsIhTI!5(7)R%B04{Oy{D(NXoJ_T%p<8uMAY!F!3ll;(<#&V!H=~3kfdbhC|S-70%{G`}CVmiQ9 zlxc-fAo9vmW+t!UC8;>f zxqY&J=N-|6WMeuKYBJJt z?k_^=iR~|BlilWqFCMoX0VYLgITI#@aVUn1a!|Ir4bQgh+!k2@?{)b zwmexWB=;T86oEMtVCpZVovDMm^X^Fah1a`j(XY^3JR@R~;B0$Pv*%J=%H@P8cS5g3 zlav`KhT&kqIpU zTlRISl#X&gqP!}GQk$#foWU+pI*&~g-mBwyO@{NSJtW+i-cr1LO&b3-gg18HG!-#Cb%H zi_I~%XH!!CtVCjYC`29{n&>FX+^b0(Rwqs5K(S6NY%yu9LKJmGVLrY>RV=S8N9$Tm Ry>wWDAFEd!i6QHN{{f`@=BNMw literal 0 HcmV?d00001 diff --git a/sdk/libraries/x64/tbbmalloc.lib b/sdk/libraries/x64/tbbmalloc.lib new file mode 100644 index 0000000000000000000000000000000000000000..8edace87bafe4e685596456c23e06f7be5103a11 GIT binary patch literal 9616 zcmc&)NpBQK6nkiDfRhkLap=KW%gt8%FHzVd9M=yW&wJJ0Qyz{PCNk^`$*}`CO}AUu|ztXqBQ;z zfY9k)M$fQBO1+>oK8O8CV~-iNVu>`Gp_Dp@d`P3aluk_pg!CFqq`_`R*guFg`Y5cj@NZt@RD}`r_?t>q~1Jx7OXt=4LTl zCLft*3}lJL$m?uIJynQ%XyS}lcY>hc=CE4v zol>#jI6uy>EzaM)=@*02qc!YwCLH(1+Ty|;Y$Elwppef!!dB-ewgVf}AxpczOmGla z=f9s~78gsUpk#!BXd{4Qp*2xC#q9azN-2265xc8<;jj{*y6MK}s!{O|5#{c(HcVR? zYJqGLu0IAkF{$adlc3s5e%Y_oP_L=BVCQc+YNo`p5 zX;7_O&FX}~u(dF?nfLtdN`6Z$>ByRkSrxW2)IxaB@kwx(Z46k7B#*%|A_hB!LD*0a zz)By$lM?_R#{f3Z0K7#?odtL~4$ylV;2F{jjKb!y{up^$kuswI=df)T%V|{f8mW5_ z>qs6(b?q1#eZX?}Bt}6<4+j9=_hUH>@ErRv_=6D`hF<7{6EFZLp&w4c5DY>Kv_d!Z zz%e)uT?U?N{~ z0$OVv&Uk#XKZMfScX#e1!Oq7RT8*?ayOH)7HDaXP7m|?ZQ}5b(Qbfg|+oq!xTD5VC zb`GsZZboEnE_#ImhGu(Z#L|~}%IsdJ_p}W@;d877FovPeMbF>-`TqEAR)uT$<9A+R zr2}0H7xP}Wk{zGqU6>++zmWcA4Q*IDaQO}6!^{_bj$+Y*_BbDDbs+Vo70X_XtjKdj z8Ps{`>FU7Nzr#Mj$g2xp<-2d@bETjhY*kWUtY1j2EUhlHex*2!H3EeYhKS*>F8sSM z{H53d2WNW+@xIEpw0!y`%OEp`h=lmC9e@z>1Ktu2kXm9qQ-lYFK=7oiEu?*n{9zB# z4)6`Dk)jpu;@{>c^E#D5TdbJL^d!rsk`>b$u9!bVfCMH$+=}sNm-s8Km@B-Ite8J| zg_UH*e5RFPGrM?eK#v;|q>24D%d^nco%RnJpi>?1w;DSw&0R%yD%olB^t_$dTeBu7?nT&$o0rDlO`VzT;9lIPM#Y;790CdS(e z%;a-DGZR18B+t+;P+JA^jUvg!Bl#MU>=Q^xL?RMG(G)7nIH^#Sn#D|J_>rfrvN=Su zUm&r)c_`Ksuc}&DIs_IPnQ^}mCqzo#5#sF#yBT2q8S6C8AQ}6A=)!h(x7Li4LkZ0;y8pj_PjJUYTvL* zCeZ%j+V=TPp+N*`!%2c4vBP|4j0eJtj%rq^S)3Ya!?@3*)uZv_rWYv1Q1TOen!k3m z)9M1+Jr-}bS$;OWjWOMgx0`nPJ1l<8#_!Y3@N$j0JsxM1U0PKqE42pawmZ&hlRP}l zFD=^Keu*CLQSFKwJ-_WMV@&+sskNGW4VJ@Pf@(JLDVjFCd&xJKKG`lyRjB5an#H_M z8CDa1Xu+>5(h6?I%B?Z7e%V%2R_v7FG$NP70mWMQ&lrm1lAl4fPTWD+nwBF{%_=pE R*-INvK5|JLlC2tcH1 literal 0 HcmV?d00001 diff --git a/sdk/libraries/x64/tbbmalloc_proxy.lib b/sdk/libraries/x64/tbbmalloc_proxy.lib new file mode 100644 index 0000000000000000000000000000000000000000..0bf4375e3c0d61782e84229ccf94da366cbdfe64 GIT binary patch literal 2192 zcmcIlPj3=I6#p%MS|r90>w#!YFsW%h2o%_=i7^Y-L{dr{O3%Z>VoS2P1ToQ*x1Rg} z{RF)G{>)9#wb6PO)A@y;Kr%tA4v@o2_=Y{eB>9=a@F_bbDf| zgKts=!fS+ol{c|4LWOIDq6E=-2)K#>=@sCN@Jhy9*bsq+5TYcb2v5Jwf4~@Up*hgG z0g9P#l9>Q&CYuygsYZO_z0;O$BQu)!=r?OWJ*P{aHsaQ47(;3gG~yz?1Glu*Zhkyo zKlC)9OoOjVk8# zEmpA>aM2B{C5RgM;d%gfkibRf?p~+WcRo1p2C2>3lT@W#-IeoW;x=g<5O+#Nca)$n zlP`>b(_plmZjWNW=xEw-=wC@oh?LwF1P0)Sf1)hzWxeyHPl0`C5k@}xwT11Q{E>}! z2Z~Z-5kix`%Pki1MVXY-pZk~;a+&nq4VFU=W|HMijlMo6<&}{c@=YmHGLx3dka^zw zAq8_|Fxjfzpg|0nR=zZ4|U A^8f$< literal 0 HcmV?d00001 diff --git a/src/3rd party/oneapi/tbb.h b/src/3rd party/oneapi/tbb.h new file mode 100644 index 0000000000..b6c871876b --- /dev/null +++ b/src/3rd party/oneapi/tbb.h @@ -0,0 +1,78 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_tbb_H +#define __TBB_tbb_H + +/** + This header bulk-includes declarations or definitions of all the functionality + provided by TBB (save for tbbmalloc and 3rd party dependent headers). + + If you use only a few TBB constructs, consider including specific headers only. + Any header listed below can be included independently of others. +**/ + +#include "oneapi/tbb/blocked_range.h" +#include "oneapi/tbb/blocked_range2d.h" +#include "oneapi/tbb/blocked_range3d.h" +#include "oneapi/tbb/blocked_nd_range.h" +#include "oneapi/tbb/cache_aligned_allocator.h" +#include "oneapi/tbb/combinable.h" +#include "oneapi/tbb/concurrent_hash_map.h" +#if TBB_PREVIEW_CONCURRENT_LRU_CACHE +#include "tbb/concurrent_lru_cache.h" +#endif +#include "oneapi/tbb/collaborative_call_once.h" +#include "oneapi/tbb/concurrent_priority_queue.h" +#include "oneapi/tbb/concurrent_queue.h" +#include "oneapi/tbb/concurrent_unordered_map.h" +#include "oneapi/tbb/concurrent_unordered_set.h" +#include "oneapi/tbb/concurrent_map.h" +#include "oneapi/tbb/concurrent_set.h" +#include "oneapi/tbb/concurrent_vector.h" +#include "oneapi/tbb/enumerable_thread_specific.h" +#include "oneapi/tbb/flow_graph.h" +#include "oneapi/tbb/global_control.h" +#include "oneapi/tbb/info.h" +#if TBB_PREVIEW_MEMORY_POOL +#include "oneapi/tbb/memory_pool.h" +#endif +#include "oneapi/tbb/null_mutex.h" +#include "oneapi/tbb/null_rw_mutex.h" +#include "oneapi/tbb/parallel_for.h" +#include "oneapi/tbb/parallel_for_each.h" +#include "oneapi/tbb/parallel_invoke.h" +#include "oneapi/tbb/parallel_pipeline.h" +#include "oneapi/tbb/parallel_reduce.h" +#include "oneapi/tbb/parallel_scan.h" +#include "oneapi/tbb/parallel_sort.h" +#include "oneapi/tbb/partitioner.h" +#include "oneapi/tbb/queuing_mutex.h" +#include "oneapi/tbb/queuing_rw_mutex.h" +#include "oneapi/tbb/scalable_allocator.h" +#include "oneapi/tbb/spin_mutex.h" +#include "oneapi/tbb/spin_rw_mutex.h" +#include "oneapi/tbb/mutex.h" +#include "oneapi/tbb/rw_mutex.h" +#include "oneapi/tbb/task.h" +#include "oneapi/tbb/task_arena.h" +#include "oneapi/tbb/task_group.h" +#include "oneapi/tbb/task_scheduler_observer.h" +#include "oneapi/tbb/tbb_allocator.h" +#include "oneapi/tbb/tick_count.h" +#include "oneapi/tbb/version.h" + +#endif /* __TBB_tbb_H */ diff --git a/src/3rd party/oneapi/tbb/blocked_nd_range.h b/src/3rd party/oneapi/tbb/blocked_nd_range.h new file mode 100644 index 0000000000..ec29917535 --- /dev/null +++ b/src/3rd party/oneapi/tbb/blocked_nd_range.h @@ -0,0 +1,196 @@ +/* + Copyright (c) 2017-2025 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_blocked_nd_range_H +#define __TBB_blocked_nd_range_H + +#include // std::any_of +#include +#include +#include // std::is_same, std::enable_if + +#include "detail/_config.h" +#include "detail/_template_helpers.h" // index_sequence, make_index_sequence +#include "detail/_namespace_injection.h" +#include "detail/_range_common.h" + +#include "blocked_range.h" + +namespace tbb { +namespace detail { +namespace d1 { + +/* + The blocked_nd_range_impl uses make_index_sequence to automatically generate a ctor with + exactly N arguments of the type tbb::blocked_range. Such ctor provides an opportunity + to use braced-init-list parameters to initialize each dimension. + Use of parameters, whose representation is a braced-init-list, but they're not + std::initializer_list or a reference to one, produces a non-deduced context + within template argument deduction. + + NOTE: blocked_nd_range must be exactly a templated alias to the blocked_nd_range_impl + (and not e.g. a derived class), otherwise it would need to declare its own ctor + facing the same problem that the impl class solves. +*/ + +template> +class blocked_nd_range_impl; + +template +class blocked_nd_range_impl> { +public: + //! Type of a value. + using value_type = Value; + + //! Type of a dimension range. + using dim_range_type = tbb::blocked_range; + + //! Type for the size of a range. + using size_type = typename dim_range_type::size_type; + + blocked_nd_range_impl() = delete; + + //! Constructs N-dimensional range over N half-open intervals each represented as tbb::blocked_range. + blocked_nd_range_impl(const indexed_t&... args) : my_dims{ {args...} } {} + +#if __clang__ && __TBB_CLANG_VERSION < 140000 + // On clang prior to version 14.0.0, passing a single braced init list to the constructor of blocked_nd_range + // matches better on the C array constructor and generates compile-time error because of unexpected size + // Adding constraints for this constructor to force the compiler to drop it from overload resolution if the size is unexpected + template ::type> + blocked_nd_range_impl(const value_type (&size)[M], size_type grainsize = 1) : +#else + blocked_nd_range_impl(const value_type (&size)[N], size_type grainsize = 1) : +#endif + my_dims { dim_range_type(0, size[Is], grainsize)... } {} + + //! Dimensionality of a range. + static constexpr unsigned int dim_count() { return N; } + + //! Range in certain dimension. + const dim_range_type& dim(unsigned int dimension) const { + __TBB_ASSERT(dimension < N, "out of bound"); + return my_dims[dimension]; + } + + //------------------------------------------------------------------------ + // Methods that implement Range concept + //------------------------------------------------------------------------ + + //! True if at least one dimension is empty. + bool empty() const { + return std::any_of(my_dims.begin(), my_dims.end(), [](const dim_range_type& d) { + return d.empty(); + }); + } + + //! True if at least one dimension is divisible. + bool is_divisible() const { + return std::any_of(my_dims.begin(), my_dims.end(), [](const dim_range_type& d) { + return d.is_divisible(); + }); + } + + blocked_nd_range_impl(blocked_nd_range_impl& r, proportional_split proportion) : my_dims(r.my_dims) { + do_split(r, proportion); + } + + blocked_nd_range_impl(blocked_nd_range_impl& r, split proportion) : my_dims(r.my_dims) { + do_split(r, proportion); + } + +private: + static_assert(N != 0, "zero dimensional blocked_nd_range can't be constructed"); + + //! Ranges in each dimension. + std::array my_dims; + + template + void do_split(blocked_nd_range_impl& r, split_type proportion) { + static_assert((std::is_same::value || std::is_same::value), + "type of split object is incorrect"); + __TBB_ASSERT(r.is_divisible(), "can't split not divisible range"); + + auto my_it = std::max_element(my_dims.begin(), my_dims.end(), [](const dim_range_type& first, const dim_range_type& second) { + return (first.size() * double(second.grainsize()) < second.size() * double(first.grainsize())); + }); + + auto r_it = r.my_dims.begin() + (my_it - my_dims.begin()); + + my_it->my_begin = dim_range_type::do_split(*r_it, proportion); + + // (!(my_it->my_begin < r_it->my_end) && !(r_it->my_end < my_it->my_begin)) equals to + // (my_it->my_begin == r_it->my_end), but we can't use operator== due to Value concept + __TBB_ASSERT(!(my_it->my_begin < r_it->my_end) && !(r_it->my_end < my_it->my_begin), + "blocked_range has been split incorrectly"); + } +}; + +template + __TBB_requires(blocked_range_value) +class blocked_nd_range : public blocked_nd_range_impl { + using base = blocked_nd_range_impl; + // Making constructors of base class visible + using base::base; +}; + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +// blocked_nd_range(const dim_range_type& dim0, const dim_range_type& dim1, ...) +// while the arguments are passed as braced-init-lists +// Works only for 2 and more arguments since the deduction from +// single braced-init-list or single C-array argument prefers the multi-dimensional range +// Only braced-init-lists of size 2 and 3 are allowed since dim_range_type may only +// be constructed from 2 or 3 arguments +template = 2>, + typename = std::enable_if_t<(... && (Ns == 2 || Ns == 3))>> +blocked_nd_range(const Value (&... dim)[Ns]) +-> blocked_nd_range; + +// blocked_nd_range(const dim_range_type& dim0, const dim_range_type& dim1, ...) +// while the arguments are passed as blocked_range objects of the same type +template )>> +blocked_nd_range(blocked_range, blocked_range...) +-> blocked_nd_range; + +// blocked_nd_range(const value_type (&size)[N], size_type grainsize = 1) +template +blocked_nd_range(const Value (&)[N], typename blocked_nd_range::size_type = 1) +-> blocked_nd_range; + +// blocked_nd_range(blocked_nd_range&, oneapi::tbb::split) +template +blocked_nd_range(blocked_nd_range, oneapi::tbb::split) +-> blocked_nd_range; + +// blocked_nd_range(blocked_nd_range&, oneapi::tbb::proportional_split) +template +blocked_nd_range(blocked_nd_range, oneapi::tbb::proportional_split) +-> blocked_nd_range; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::blocked_nd_range; +} // namespace v1 +} // namespace tbb + +#endif /* __TBB_blocked_nd_range_H */ diff --git a/src/3rd party/oneapi/tbb/blocked_range.h b/src/3rd party/oneapi/tbb/blocked_range.h new file mode 100644 index 0000000000..30168e9055 --- /dev/null +++ b/src/3rd party/oneapi/tbb/blocked_range.h @@ -0,0 +1,168 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_blocked_range_H +#define __TBB_blocked_range_H + +#include + +#include "detail/_range_common.h" +#include "detail/_namespace_injection.h" + +namespace tbb { +namespace detail { +namespace d1 { + +/** \page range_req Requirements on range concept + Class \c R implementing the concept of range must define: + - \code R::R( const R& ); \endcode Copy constructor + - \code R::~R(); \endcode Destructor + - \code bool R::is_divisible() const; \endcode True if range can be partitioned into two subranges + - \code bool R::empty() const; \endcode True if range is empty + - \code R::R( R& r, split ); \endcode Split range \c r into two subranges. +**/ + +//! A range over which to iterate. +/** @ingroup algorithms */ +template + __TBB_requires(blocked_range_value) +class blocked_range { +public: + //! Type of a value + /** Called a const_iterator for sake of algorithms that need to treat a blocked_range + as an STL container. */ + using const_iterator = Value; + + //! Type for size of a range + using size_type = std::size_t; + + //! Construct range over half-open interval [begin,end), with the given grainsize. + blocked_range( Value begin_, Value end_, size_type grainsize_=1 ) : + my_end(end_), my_begin(begin_), my_grainsize(grainsize_) + { + __TBB_ASSERT( my_grainsize>0, "grainsize must be positive" ); + } + + //! Beginning of range. + const_iterator begin() const { return my_begin; } + + //! One past last value in range. + const_iterator end() const { return my_end; } + + //! Size of the range + /** Unspecified if end() + __TBB_requires(blocked_range_value && + blocked_range_value) + friend class blocked_range2d; + + template + __TBB_requires(blocked_range_value && + blocked_range_value && + blocked_range_value) + friend class blocked_range3d; + + template + friend class blocked_nd_range_impl; +}; + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::blocked_range; +// Split types +using detail::split; +using detail::proportional_split; +} // namespace v1 + +} // namespace tbb + +#endif /* __TBB_blocked_range_H */ diff --git a/src/3rd party/oneapi/tbb/blocked_range2d.h b/src/3rd party/oneapi/tbb/blocked_range2d.h new file mode 100644 index 0000000000..41385db98e --- /dev/null +++ b/src/3rd party/oneapi/tbb/blocked_range2d.h @@ -0,0 +1,111 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_blocked_range2d_H +#define __TBB_blocked_range2d_H + +#include + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_range_common.h" + +#include "blocked_range.h" + +namespace tbb { +namespace detail { +namespace d1 { + +//! A 2-dimensional range that models the Range concept. +/** @ingroup algorithms */ +template + __TBB_requires(blocked_range_value && + blocked_range_value) +class blocked_range2d { +public: + //! Type for size of an iteration range + using row_range_type = blocked_range; + using col_range_type = blocked_range; + +private: + row_range_type my_rows; + col_range_type my_cols; + +public: + blocked_range2d( RowValue row_begin, RowValue row_end, typename row_range_type::size_type row_grainsize, + ColValue col_begin, ColValue col_end, typename col_range_type::size_type col_grainsize ) : + my_rows(row_begin,row_end,row_grainsize), + my_cols(col_begin,col_end,col_grainsize) + {} + + blocked_range2d( RowValue row_begin, RowValue row_end, + ColValue col_begin, ColValue col_end ) : + my_rows(row_begin,row_end), + my_cols(col_begin,col_end) + {} + + //! True if range is empty + bool empty() const { + // Range is empty if at least one dimension is empty. + return my_rows.empty() || my_cols.empty(); + } + + //! True if range is divisible into two pieces. + bool is_divisible() const { + return my_rows.is_divisible() || my_cols.is_divisible(); + } + + blocked_range2d( blocked_range2d& r, split ) : + my_rows(r.my_rows), + my_cols(r.my_cols) + { + split split_obj; + do_split(r, split_obj); + } + + blocked_range2d( blocked_range2d& r, proportional_split& proportion ) : + my_rows(r.my_rows), + my_cols(r.my_cols) + { + do_split(r, proportion); + } + + //! The rows of the iteration space + const row_range_type& rows() const { return my_rows; } + + //! The columns of the iteration space + const col_range_type& cols() const { return my_cols; } + +private: + template + void do_split( blocked_range2d& r, Split& split_obj ) { + if ( my_rows.size()*double(my_cols.grainsize()) < my_cols.size()*double(my_rows.grainsize()) ) { + my_cols.my_begin = col_range_type::do_split(r.my_cols, split_obj); + } else { + my_rows.my_begin = row_range_type::do_split(r.my_rows, split_obj); + } + } +}; + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::blocked_range2d; +} // namespace v1 +} // namespace tbb + +#endif /* __TBB_blocked_range2d_H */ diff --git a/src/3rd party/oneapi/tbb/blocked_range3d.h b/src/3rd party/oneapi/tbb/blocked_range3d.h new file mode 100644 index 0000000000..d8932192b4 --- /dev/null +++ b/src/3rd party/oneapi/tbb/blocked_range3d.h @@ -0,0 +1,130 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_blocked_range3d_H +#define __TBB_blocked_range3d_H + +#include + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" + +#include "blocked_range.h" + +namespace tbb { +namespace detail { +namespace d1 { + +//! A 3-dimensional range that models the Range concept. +/** @ingroup algorithms */ +template + __TBB_requires(blocked_range_value && + blocked_range_value && + blocked_range_value) +class blocked_range3d { +public: + //! Type for size of an iteration range + using page_range_type = blocked_range; + using row_range_type = blocked_range; + using col_range_type = blocked_range; + +private: + page_range_type my_pages; + row_range_type my_rows; + col_range_type my_cols; + +public: + + blocked_range3d( PageValue page_begin, PageValue page_end, + RowValue row_begin, RowValue row_end, + ColValue col_begin, ColValue col_end ) : + my_pages(page_begin,page_end), + my_rows(row_begin,row_end), + my_cols(col_begin,col_end) + {} + + blocked_range3d( PageValue page_begin, PageValue page_end, typename page_range_type::size_type page_grainsize, + RowValue row_begin, RowValue row_end, typename row_range_type::size_type row_grainsize, + ColValue col_begin, ColValue col_end, typename col_range_type::size_type col_grainsize ) : + my_pages(page_begin,page_end,page_grainsize), + my_rows(row_begin,row_end,row_grainsize), + my_cols(col_begin,col_end,col_grainsize) + {} + + //! True if range is empty + bool empty() const { + // Range is empty if at least one dimension is empty. + return my_pages.empty() || my_rows.empty() || my_cols.empty(); + } + + //! True if range is divisible into two pieces. + bool is_divisible() const { + return my_pages.is_divisible() || my_rows.is_divisible() || my_cols.is_divisible(); + } + + blocked_range3d( blocked_range3d& r, split split_obj ) : + my_pages(r.my_pages), + my_rows(r.my_rows), + my_cols(r.my_cols) + { + do_split(r, split_obj); + } + + blocked_range3d( blocked_range3d& r, proportional_split& proportion ) : + my_pages(r.my_pages), + my_rows(r.my_rows), + my_cols(r.my_cols) + { + do_split(r, proportion); + } + + //! The pages of the iteration space + const page_range_type& pages() const { return my_pages; } + + //! The rows of the iteration space + const row_range_type& rows() const { return my_rows; } + + //! The columns of the iteration space + const col_range_type& cols() const { return my_cols; } + +private: + template + void do_split( blocked_range3d& r, Split& split_obj) { + if ( my_pages.size()*double(my_rows.grainsize()) < my_rows.size()*double(my_pages.grainsize()) ) { + if ( my_rows.size()*double(my_cols.grainsize()) < my_cols.size()*double(my_rows.grainsize()) ) { + my_cols.my_begin = col_range_type::do_split(r.my_cols, split_obj); + } else { + my_rows.my_begin = row_range_type::do_split(r.my_rows, split_obj); + } + } else { + if ( my_pages.size()*double(my_cols.grainsize()) < my_cols.size()*double(my_pages.grainsize()) ) { + my_cols.my_begin = col_range_type::do_split(r.my_cols, split_obj); + } else { + my_pages.my_begin = page_range_type::do_split(r.my_pages, split_obj); + } + } + } +}; + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::blocked_range3d; +} // namespace v1 +} // namespace tbb + +#endif /* __TBB_blocked_range3d_H */ diff --git a/src/3rd party/oneapi/tbb/cache_aligned_allocator.h b/src/3rd party/oneapi/tbb/cache_aligned_allocator.h new file mode 100644 index 0000000000..0ff3972d25 --- /dev/null +++ b/src/3rd party/oneapi/tbb/cache_aligned_allocator.h @@ -0,0 +1,189 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_cache_aligned_allocator_H +#define __TBB_cache_aligned_allocator_H + +#include "detail/_utils.h" +#include "detail/_namespace_injection.h" +#include +#include + +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT +#include +#endif + +namespace tbb { +namespace detail { + +namespace r1 { +TBB_EXPORT void* __TBB_EXPORTED_FUNC cache_aligned_allocate(std::size_t size); +TBB_EXPORT void __TBB_EXPORTED_FUNC cache_aligned_deallocate(void* p); +TBB_EXPORT std::size_t __TBB_EXPORTED_FUNC cache_line_size(); +} + +namespace d1 { + +template +class cache_aligned_allocator { +public: + using value_type = T; + using propagate_on_container_move_assignment = std::true_type; + + //! Always defined for TBB containers (supported since C++17 for std containers) + using is_always_equal = std::true_type; + + cache_aligned_allocator() = default; + template cache_aligned_allocator(const cache_aligned_allocator&) noexcept {} + + //! Allocate space for n objects, starting on a cache/sector line. + __TBB_nodiscard T* allocate(std::size_t n) { + return static_cast(r1::cache_aligned_allocate(n * sizeof(value_type))); + } + + //! Free block of memory that starts on a cache line + void deallocate(T* p, std::size_t) { + r1::cache_aligned_deallocate(p); + } + + //! Largest value for which method allocate might succeed. + std::size_t max_size() const noexcept { + return (~std::size_t(0) - r1::cache_line_size()) / sizeof(value_type); + } + +#if TBB_ALLOCATOR_TRAITS_BROKEN + using pointer = value_type*; + using const_pointer = const value_type*; + using reference = value_type&; + using const_reference = const value_type&; + using difference_type = std::ptrdiff_t; + using size_type = std::size_t; + template struct rebind { + using other = cache_aligned_allocator; + }; + template + void construct(U *p, Args&&... args) + { ::new (p) U(std::forward(args)...); } + void destroy(pointer p) { p->~value_type(); } + pointer address(reference x) const { return &x; } + const_pointer address(const_reference x) const { return &x; } +#endif // TBB_ALLOCATOR_TRAITS_BROKEN +}; + +#if TBB_ALLOCATOR_TRAITS_BROKEN + template<> + class cache_aligned_allocator { + public: + using pointer = void*; + using const_pointer = const void*; + using value_type = void; + template struct rebind { + using other = cache_aligned_allocator; + }; + }; +#endif + +template +bool operator==(const cache_aligned_allocator&, const cache_aligned_allocator&) noexcept { return true; } + +#if !__TBB_CPP20_COMPARISONS_PRESENT +template +bool operator!=(const cache_aligned_allocator&, const cache_aligned_allocator&) noexcept { return false; } +#endif + +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT + +//! C++17 memory resource wrapper to ensure cache line size alignment +class cache_aligned_resource : public std::pmr::memory_resource { +public: + cache_aligned_resource() : cache_aligned_resource(std::pmr::get_default_resource()) {} + explicit cache_aligned_resource(std::pmr::memory_resource* upstream) : m_upstream(upstream) {} + + std::pmr::memory_resource* upstream_resource() const { + return m_upstream; + } + +private: + //! We don't know what memory resource set. Use padding to guarantee alignment + void* do_allocate(std::size_t bytes, std::size_t alignment) override { + // TODO: make it common with tbb_allocator.cpp + std::size_t cache_line_alignment = correct_alignment(alignment); + std::size_t space = correct_size(bytes) + cache_line_alignment; + std::uintptr_t base = reinterpret_cast(m_upstream->allocate(space)); + __TBB_ASSERT(base != 0, "Upstream resource returned nullptr."); + + // Round up to the next cache line (align the base address) + std::uintptr_t result = (base + cache_line_alignment) & ~(cache_line_alignment - 1); + __TBB_ASSERT((result - base) >= sizeof(std::uintptr_t), "Can`t store a base pointer to the header"); + __TBB_ASSERT(space - (result - base) >= bytes, "Not enough space for the storage"); + + // Record where block actually starts. + (reinterpret_cast(result))[-1] = base; + return reinterpret_cast(result); + } + + void do_deallocate(void* ptr, std::size_t bytes, std::size_t alignment) override { + if (ptr) { + // Recover where block actually starts + std::uintptr_t base = (reinterpret_cast(ptr))[-1]; + m_upstream->deallocate(reinterpret_cast(base), correct_size(bytes) + correct_alignment(alignment)); + } + } + + bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override { + if (this == &other) { return true; } +#if __TBB_USE_OPTIONAL_RTTI + const cache_aligned_resource* other_res = dynamic_cast(&other); + return other_res && (upstream_resource() == other_res->upstream_resource()); +#else + return false; +#endif + } + + std::size_t correct_alignment(std::size_t alignment) { + __TBB_ASSERT(tbb::detail::is_power_of_two(alignment), "Alignment is not a power of 2"); +#if __TBB_CPP17_HW_INTERFERENCE_SIZE_PRESENT + std::size_t cache_line_size = std::hardware_destructive_interference_size; +#else + std::size_t cache_line_size = r1::cache_line_size(); +#endif + return alignment < cache_line_size ? cache_line_size : alignment; + } + + std::size_t correct_size(std::size_t bytes) { + // To handle the case, when small size requested. There could be not + // enough space to store the original pointer. + return bytes < sizeof(std::uintptr_t) ? sizeof(std::uintptr_t) : bytes; + } + + std::pmr::memory_resource* m_upstream; +}; + +#endif // __TBB_CPP17_MEMORY_RESOURCE_PRESENT + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::cache_aligned_allocator; +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT +using detail::d1::cache_aligned_resource; +#endif +} // namespace v1 +} // namespace tbb + +#endif /* __TBB_cache_aligned_allocator_H */ + diff --git a/src/3rd party/oneapi/tbb/collaborative_call_once.h b/src/3rd party/oneapi/tbb/collaborative_call_once.h new file mode 100644 index 0000000000..b666a66e11 --- /dev/null +++ b/src/3rd party/oneapi/tbb/collaborative_call_once.h @@ -0,0 +1,257 @@ +/* + Copyright (c) 2021-2024 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_collaborative_call_once_H +#define __TBB_collaborative_call_once_H + +#include "task_arena.h" +#include "task_group.h" + +#include + +namespace tbb { +namespace detail { +namespace d1 { + +#if _MSC_VER && !defined(__INTEL_COMPILER) + // Suppress warning: structure was padded due to alignment specifier + #pragma warning (push) + #pragma warning (disable: 4324) +#endif + +template +class collaborative_call_stack_task : public task { + const F& m_func; + wait_context& m_wait_ctx; + + void finalize() { + m_wait_ctx.release(); + } + task* execute(d1::execution_data&) override { + task* res = d2::task_ptr_or_nullptr(m_func); + finalize(); + return res; + } + task* cancel(d1::execution_data&) override { + finalize(); + return nullptr; + } +public: + collaborative_call_stack_task(const F& f, wait_context& wctx) : m_func(f), m_wait_ctx(wctx) {} +}; + +__TBB_GLOBAL_VAR constexpr std::uintptr_t collaborative_once_max_references = max_nfs_size; +__TBB_GLOBAL_VAR constexpr std::uintptr_t collaborative_once_references_mask = collaborative_once_max_references-1; + +class alignas(max_nfs_size) collaborative_once_runner : no_copy { + + struct storage_t { + task_arena m_arena{ task_arena::attach{} }; + wait_context m_wait_context{1}; + }; + + std::atomic m_ref_count{0}; + std::atomic m_is_ready{false}; + + // Storage with task_arena and wait_context must be initialized only by winner thread + union { + storage_t m_storage; + }; + + template + void isolated_execute(Fn f) { + auto func = [f] { + f(); + // delegate_base requires bool returning functor while isolate_within_arena ignores the result + return true; + }; + + delegated_function delegate(func); + + r1::isolate_within_arena(delegate, reinterpret_cast(this)); + } + +public: + class lifetime_guard : no_copy { + collaborative_once_runner& m_runner; + public: + lifetime_guard(collaborative_once_runner& r) : m_runner(r) { + m_runner.m_ref_count++; + } + ~lifetime_guard() { + m_runner.m_ref_count--; + } + }; + + collaborative_once_runner() {} + + ~collaborative_once_runner() { + spin_wait_until_eq(m_ref_count, 0, std::memory_order_acquire); + if (m_is_ready.load(std::memory_order_relaxed)) { + m_storage.~storage_t(); + } + } + + std::uintptr_t to_bits() { + return reinterpret_cast(this); + } + + static collaborative_once_runner* from_bits(std::uintptr_t bits) { + __TBB_ASSERT( (bits & collaborative_once_references_mask) == 0, "invalid pointer, last log2(max_nfs_size) bits must be zero" ); + return reinterpret_cast(bits); + } + + template + void run_once(F&& f) { + __TBB_ASSERT(!m_is_ready.load(std::memory_order_relaxed), "storage with task_arena and wait_context is already initialized"); + // Initialize internal state + new(&m_storage) storage_t(); + m_storage.m_arena.execute([&] { + isolated_execute([&] { + task_group_context context{ task_group_context::bound, + task_group_context::default_traits | task_group_context::concurrent_wait }; + + collaborative_call_stack_task t{ std::forward(f), m_storage.m_wait_context }; + + // Set the ready flag after entering the execute body to prevent + // moonlighting threads from occupying all slots inside the arena. + m_is_ready.store(true, std::memory_order_release); + execute_and_wait(t, context, m_storage.m_wait_context, context); + }); + }); + } + + void assist() noexcept { + // Do not join the arena until the winner thread takes the slot + spin_wait_while_eq(m_is_ready, false); + m_storage.m_arena.execute([&] { + isolated_execute([&] { + // We do not want to get an exception from user functor on moonlighting threads. + // The exception is handled with the winner thread + task_group_context stub_context; + wait(m_storage.m_wait_context, stub_context); + }); + }); + } + +}; + +class collaborative_once_flag : no_copy { + enum state : std::uintptr_t { + uninitialized, + done, +#if TBB_USE_ASSERT + dead +#endif + }; + std::atomic m_state{ state::uninitialized }; + + template + friend void collaborative_call_once(collaborative_once_flag& flag, Fn&& f, Args&&... args); + + void set_completion_state(std::uintptr_t runner_bits, std::uintptr_t desired) { + std::uintptr_t expected = runner_bits; + do { + expected = runner_bits; + // Possible inefficiency: when we start waiting, + // some moonlighting threads might continue coming that will prolong our waiting. + // Fortunately, there are limited number of threads on the system so wait time is limited. + spin_wait_until_eq(m_state, expected); + } while (!m_state.compare_exchange_strong(expected, desired)); + } + + template + void do_collaborative_call_once(Fn&& f) { + std::uintptr_t expected = m_state.load(std::memory_order_acquire); + collaborative_once_runner runner; + + do { + if (expected == state::uninitialized && m_state.compare_exchange_strong(expected, runner.to_bits())) { + // Winner thread + runner.run_once([&] { + try_call([&] { + std::forward(f)(); + }).on_exception([&] { + // Reset the state to uninitialized to allow other threads to try initialization again + set_completion_state(runner.to_bits(), state::uninitialized); + }); + // We successfully executed functor + set_completion_state(runner.to_bits(), state::done); + }); + break; + } else { + // Moonlighting thread: we need to add a reference to the state to prolong runner lifetime. + // However, the maximum number of references are limited with runner alignment. + // So, we use CAS loop and spin_wait to guarantee that references never exceed "max_value". + do { + auto max_value = expected | collaborative_once_references_mask; + expected = spin_wait_while_eq(m_state, max_value); + // "expected > state::done" prevents storing values, when state is uninitialized or done + } while (expected > state::done && !m_state.compare_exchange_strong(expected, expected + 1)); + + if (auto shared_runner = collaborative_once_runner::from_bits(expected & ~collaborative_once_references_mask)) { + collaborative_once_runner::lifetime_guard guard{*shared_runner}; + m_state.fetch_sub(1); + + // The moonlighting threads are not expected to handle exceptions from user functor. + // Therefore, no exception is expected from assist(). + shared_runner->assist(); + } + } + __TBB_ASSERT(m_state.load(std::memory_order_relaxed) != state::dead, + "collaborative_once_flag has been prematurely destroyed"); + } while (expected != state::done); + } + +#if TBB_USE_ASSERT +public: + ~collaborative_once_flag() { + m_state.store(state::dead, std::memory_order_relaxed); + } +#endif +}; + + +template +void collaborative_call_once(collaborative_once_flag& flag, Fn&& fn, Args&&... args) { + __TBB_ASSERT(flag.m_state.load(std::memory_order_relaxed) != collaborative_once_flag::dead, + "collaborative_once_flag has been prematurely destroyed"); + if (flag.m_state.load(std::memory_order_acquire) != collaborative_once_flag::done) { + #if __TBB_GCC_PARAMETER_PACK_IN_LAMBDAS_BROKEN + // Using stored_pack to suppress bug in GCC 4.8 + // with parameter pack expansion in lambda + auto stored_pack = save_pack(std::forward(args)...); + auto func = [&] { call(std::forward(fn), std::move(stored_pack)); }; + #else + auto func = [&] { fn(std::forward(args)...); }; + #endif + flag.do_collaborative_call_once(func); + } +} + +#if _MSC_VER && !defined(__INTEL_COMPILER) + #pragma warning (pop) // 4324 warning +#endif + +} // namespace d1 +} // namespace detail + +using detail::d1::collaborative_call_once; +using detail::d1::collaborative_once_flag; +} // namespace tbb + +#endif // __TBB_collaborative_call_once_H diff --git a/src/3rd party/oneapi/tbb/combinable.h b/src/3rd party/oneapi/tbb/combinable.h new file mode 100644 index 0000000000..b676a30cc0 --- /dev/null +++ b/src/3rd party/oneapi/tbb/combinable.h @@ -0,0 +1,69 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_combinable_H +#define __TBB_combinable_H + +#include "detail/_namespace_injection.h" + +#include "enumerable_thread_specific.h" +#include "cache_aligned_allocator.h" + +namespace tbb { +namespace detail { +namespace d1 { +/** \name combinable **/ +//@{ +//! Thread-local storage with optional reduction +/** @ingroup containers */ +template +class combinable { + using my_alloc = typename tbb::cache_aligned_allocator; + using my_ets_type = typename tbb::enumerable_thread_specific; + my_ets_type my_ets; + +public: + combinable() = default; + + template + explicit combinable(Finit _finit) : my_ets(_finit) { } + + void clear() { my_ets.clear(); } + + T& local() { return my_ets.local(); } + + T& local(bool& exists) { return my_ets.local(exists); } + + // combine_func_t has signature T(T,T) or T(const T&, const T&) + template + T combine(CombineFunc f_combine) { return my_ets.combine(f_combine); } + + // combine_func_t has signature void(T) or void(const T&) + template + void combine_each(CombineFunc f_combine) { my_ets.combine_each(f_combine); } +}; + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::combinable; +} // inline namespace v1 + +} // namespace tbb + +#endif /* __TBB_combinable_H */ + diff --git a/src/3rd party/oneapi/tbb/concurrent_hash_map.h b/src/3rd party/oneapi/tbb/concurrent_hash_map.h new file mode 100644 index 0000000000..147e32f74e --- /dev/null +++ b/src/3rd party/oneapi/tbb/concurrent_hash_map.h @@ -0,0 +1,1671 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + Copyright (c) 2025 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_hash_map_H +#define __TBB_concurrent_hash_map_H + +#include "detail/_namespace_injection.h" +#include "detail/_utils.h" +#include "detail/_assert.h" +#include "detail/_allocator_traits.h" +#include "detail/_containers_helpers.h" +#include "detail/_template_helpers.h" +#include "detail/_hash_compare.h" +#include "detail/_range_common.h" +#include "tbb_allocator.h" +#include "spin_rw_mutex.h" + +#include +#include +#include +#include +#include // Need std::pair +#include // Need std::memset + +namespace tbb { +namespace detail { +namespace d2 { + +#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS && __TBB_CPP20_CONCEPTS_PRESENT +template +concept ch_map_rw_scoped_lockable = rw_scoped_lockable && + requires(const typename Mutex::scoped_lock& sl) { + { sl.is_writer() } -> std::convertible_to; +}; +#endif + +template +struct hash_map_node_base : no_copy { + using mutex_type = MutexType; + // Scoped lock type for mutex + using scoped_type = typename MutexType::scoped_lock; + // Next node in chain + hash_map_node_base* next; + mutex_type mutex; +}; + +// Incompleteness flag value +__TBB_GLOBAL_VAR void* const rehash_req_flag = reinterpret_cast(std::size_t(3)); +// Rehashed empty bucket flag +__TBB_GLOBAL_VAR void* const empty_rehashed_flag = reinterpret_cast(std::size_t(0)); + +template +bool rehash_required( hash_map_node_base* node_ptr ) { + return reinterpret_cast(node_ptr) == rehash_req_flag; +} + +#if TBB_USE_ASSERT +template +bool empty_rehashed( hash_map_node_base* node_ptr ) { + return reinterpret_cast(node_ptr) == empty_rehashed_flag; +} +#endif + +// base class of concurrent_hash_map + +template +class hash_map_base { +public: + using size_type = std::size_t; + using hashcode_type = std::size_t; + using segment_index_type = std::size_t; + using node_base = hash_map_node_base; + + struct bucket : no_copy { + using mutex_type = MutexType; + using scoped_type = typename mutex_type::scoped_lock; + + bucket() : node_list(nullptr) {} + bucket( node_base* ptr ) : node_list(ptr) {} + + mutex_type mutex; + std::atomic node_list; + }; + + using allocator_type = Allocator; + using allocator_traits_type = tbb::detail::allocator_traits; + using bucket_allocator_type = typename allocator_traits_type::template rebind_alloc; + using bucket_allocator_traits = tbb::detail::allocator_traits; + + // Count of segments in the first block + static constexpr size_type embedded_block = 1; + // Count of segments in the first block + static constexpr size_type embedded_buckets = 1 << embedded_block; + // Count of segments in the first block + static constexpr size_type first_block = 8; //including embedded_block. perfect with bucket size 16, so the allocations are power of 4096 + // Size of a pointer / table size + static constexpr size_type pointers_per_table = sizeof(segment_index_type) * 8; // one segment per bit + + using segment_ptr_type = bucket*; + using atomic_segment_type = std::atomic; + using segments_table_type = atomic_segment_type[pointers_per_table]; + + hash_map_base( const allocator_type& alloc ) : my_allocator(alloc), my_mask(embedded_buckets - 1), my_size(0) { + for (size_type i = 0; i != embedded_buckets; ++i) { + my_embedded_segment[i].node_list.store(nullptr, std::memory_order_relaxed); + } + + for (size_type segment_index = 0; segment_index < pointers_per_table; ++segment_index) { + auto argument = segment_index < embedded_block ? my_embedded_segment + segment_base(segment_index) : nullptr; + my_table[segment_index].store(argument, std::memory_order_relaxed); + } + + __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks"); + } + + // segment index of given index in the array + static segment_index_type segment_index_of( size_type index ) { + return segment_index_type(tbb::detail::log2( index|1 )); + } + + // the first array index of given segment + static segment_index_type segment_base( segment_index_type k ) { + return (segment_index_type(1) << k & ~segment_index_type(1)); + } + + // segment size except for k == 0 + static size_type segment_size( segment_index_type k ) { + return size_type(1) << k; // fake value for k==0 + } + + // true if ptr is valid pointer + static bool is_valid( void* ptr ) { + return reinterpret_cast(ptr) > uintptr_t(63); + } + + template + void init_buckets_impl( segment_ptr_type ptr, size_type sz, const Args&... args ) { + for (size_type i = 0; i < sz; ++i) { + bucket_allocator_traits::construct(my_allocator, ptr + i, args...); + } + } + + // Initialize buckets + void init_buckets( segment_ptr_type ptr, size_type sz, bool is_initial ) { + if (is_initial) { + init_buckets_impl(ptr, sz); + } else { + init_buckets_impl(ptr, sz, reinterpret_cast(rehash_req_flag)); + } + } + + // Add node n to bucket b + static void add_to_bucket( bucket* b, node_base* n ) { + __TBB_ASSERT(!rehash_required(b->node_list.load(std::memory_order_relaxed)), nullptr); + n->next = b->node_list.load(std::memory_order_relaxed); + b->node_list.store(n, std::memory_order_relaxed); // its under lock and flag is set + } + + const bucket_allocator_type& get_allocator() const { + return my_allocator; + } + + bucket_allocator_type& get_allocator() { + return my_allocator; + } + + // Enable segment + void enable_segment( segment_index_type k, bool is_initial = false ) { + __TBB_ASSERT( k, "Zero segment must be embedded" ); + size_type sz; + __TBB_ASSERT( !is_valid(my_table[k].load(std::memory_order_relaxed)), "Wrong concurrent assignment"); + if (k >= first_block) { + sz = segment_size(k); + segment_ptr_type ptr = nullptr; + try_call( [&] { + ptr = bucket_allocator_traits::allocate(my_allocator, sz); + } ).on_exception( [&] { + my_table[k].store(nullptr, std::memory_order_relaxed); + }); + + __TBB_ASSERT(ptr, nullptr); + init_buckets(ptr, sz, is_initial); + my_table[k].store(ptr, std::memory_order_release); + sz <<= 1;// double it to get entire capacity of the container + } else { // the first block + __TBB_ASSERT( k == embedded_block, "Wrong segment index" ); + sz = segment_size(first_block); + segment_ptr_type ptr = nullptr; + try_call( [&] { + ptr = bucket_allocator_traits::allocate(my_allocator, sz - embedded_buckets); + } ).on_exception( [&] { + my_table[k].store(nullptr, std::memory_order_relaxed); + }); + + __TBB_ASSERT(ptr, nullptr); + init_buckets(ptr, sz - embedded_buckets, is_initial); + ptr -= segment_base(embedded_block); + for(segment_index_type i = embedded_block; i < first_block; i++) // calc the offsets + my_table[i].store(ptr + segment_base(i), std::memory_order_release); + } + my_mask.store(sz-1, std::memory_order_release); + } + + void delete_segment( segment_index_type s ) { + segment_ptr_type buckets_ptr = my_table[s].load(std::memory_order_relaxed); + size_type sz = segment_size( s ? s : 1 ); + + size_type deallocate_size = 0; + + if (s >= first_block) { // the first segment or the next + deallocate_size = sz; + } else if (s == embedded_block && embedded_block != first_block) { + deallocate_size = segment_size(first_block) - embedded_buckets; + } + + for (size_type i = 0; i < deallocate_size; ++i) { + bucket_allocator_traits::destroy(my_allocator, buckets_ptr + i); + } + if (deallocate_size != 0) { + bucket_allocator_traits::deallocate(my_allocator, buckets_ptr, deallocate_size); + } + + if (s >= embedded_block) my_table[s].store(nullptr, std::memory_order_relaxed); + } + + // Get bucket by (masked) hashcode + bucket *get_bucket( hashcode_type h ) const noexcept { + segment_index_type s = segment_index_of( h ); + h -= segment_base(s); + segment_ptr_type seg = my_table[s].load(std::memory_order_acquire); + __TBB_ASSERT( is_valid(seg), "hashcode must be cut by valid mask for allocated segments" ); + return &seg[h]; + } + + // detail serial rehashing helper + void mark_rehashed_levels( hashcode_type h ) noexcept { + segment_index_type s = segment_index_of( h ); + while (segment_ptr_type seg = my_table[++s].load(std::memory_order_relaxed)) + if (rehash_required(seg[h].node_list.load(std::memory_order_relaxed))) { + seg[h].node_list.store(reinterpret_cast(empty_rehashed_flag), std::memory_order_relaxed); + mark_rehashed_levels( h + ((hashcode_type)1<node_list.load(std::memory_order_acquire))) { + return true; + } + } + return false; + } + + // Insert a node and check for load factor. @return segment index to enable. + segment_index_type insert_new_node( bucket *b, node_base *n, hashcode_type mask ) { + size_type sz = ++my_size; // prefix form is to enforce allocation after the first item inserted + add_to_bucket( b, n ); + // check load factor + if( sz >= mask ) { // TODO: add custom load_factor + segment_index_type new_seg = tbb::detail::log2( mask+1 ); //optimized segment_index_of + __TBB_ASSERT( is_valid(my_table[new_seg-1].load(std::memory_order_relaxed)), "new allocations must not publish new mask until segment has allocated"); + static const segment_ptr_type is_allocating = segment_ptr_type(2); + segment_ptr_type disabled = nullptr; + if (!(my_table[new_seg].load(std::memory_order_acquire)) + && my_table[new_seg].compare_exchange_strong(disabled, is_allocating)) + return new_seg; // The value must be processed + } + return 0; + } + + // Prepare enough segments for number of buckets + void reserve(size_type buckets) { + if( !buckets-- ) return; + bool is_initial = !my_size.load(std::memory_order_relaxed); + for (size_type m = my_mask.load(std::memory_order_relaxed); buckets > m; + m = my_mask.load(std::memory_order_relaxed)) + { + enable_segment( segment_index_of( m+1 ), is_initial ); + } + } + + // Swap hash_map_bases + void internal_swap_content(hash_map_base &table) { + using std::swap; + swap_atomics_relaxed(my_mask, table.my_mask); + swap_atomics_relaxed(my_size, table.my_size); + + for(size_type i = 0; i < embedded_buckets; i++) { + auto temp = my_embedded_segment[i].node_list.load(std::memory_order_relaxed); + my_embedded_segment[i].node_list.store(table.my_embedded_segment[i].node_list.load(std::memory_order_relaxed), + std::memory_order_relaxed); + table.my_embedded_segment[i].node_list.store(temp, std::memory_order_relaxed); + } + for(size_type i = embedded_block; i < pointers_per_table; i++) { + auto temp = my_table[i].load(std::memory_order_relaxed); + my_table[i].store(table.my_table[i].load(std::memory_order_relaxed), + std::memory_order_relaxed); + table.my_table[i].store(temp, std::memory_order_relaxed); + } + } + + void internal_move(hash_map_base&& other) { + my_mask.store(other.my_mask.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.my_mask.store(embedded_buckets - 1, std::memory_order_relaxed); + + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.my_size.store(0, std::memory_order_relaxed); + + for (size_type i = 0; i < embedded_buckets; ++i) { + my_embedded_segment[i].node_list.store(other.my_embedded_segment[i].node_list, std::memory_order_relaxed); + other.my_embedded_segment[i].node_list.store(nullptr, std::memory_order_relaxed); + } + + for (size_type i = embedded_block; i < pointers_per_table; ++i) { + my_table[i].store(other.my_table[i].load(std::memory_order_relaxed), + std::memory_order_relaxed); + other.my_table[i].store(nullptr, std::memory_order_relaxed); + } + } + +protected: + bucket_allocator_type my_allocator; + // Hash mask = sum of allocated segment sizes - 1 + std::atomic my_mask; + // Size of container in stored items + std::atomic my_size; // It must be in separate cache line from my_mask due to performance effects + // Zero segment + bucket my_embedded_segment[embedded_buckets]; + // Segment pointers table. Also prevents false sharing between my_mask and my_size + segments_table_type my_table; +}; + +template +class hash_map_range; + +// Meets requirements of a forward iterator for STL +// Value is either the T or const T type of the container. +template +class hash_map_iterator { + using map_type = Container; + using node = typename Container::node; + using map_base = typename Container::base_type; + using node_base = typename map_base::node_base; + using bucket = typename map_base::bucket; +public: + using value_type = Value; + using size_type = typename Container::size_type; + using difference_type = typename Container::difference_type; + using pointer = value_type*; + using reference = value_type&; + using iterator_category = std::forward_iterator_tag; + + // Construct undefined iterator + hash_map_iterator(): my_map(), my_index(), my_bucket(), my_node() {} + hash_map_iterator( const hash_map_iterator& other ) : + my_map(other.my_map), + my_index(other.my_index), + my_bucket(other.my_bucket), + my_node(other.my_node) + {} + + hash_map_iterator& operator=( const hash_map_iterator& other ) { + my_map = other.my_map; + my_index = other.my_index; + my_bucket = other.my_bucket; + my_node = other.my_node; + return *this; + } + + Value& operator*() const { + __TBB_ASSERT( map_base::is_valid(my_node), "iterator uninitialized or at end of container?" ); + return my_node->value(); + } + + Value* operator->() const {return &operator*();} + + hash_map_iterator& operator++() { + my_node = static_cast( my_node->next ); + if( !my_node ) advance_to_next_bucket(); + return *this; + } + + // Post increment + hash_map_iterator operator++(int) { + hash_map_iterator old(*this); + operator++(); + return old; + } +private: + template + friend bool operator==( const hash_map_iterator& i, const hash_map_iterator& j ); + + template + friend bool operator!=( const hash_map_iterator& i, const hash_map_iterator& j ); + + template + friend class hash_map_iterator; + + template + friend class hash_map_range; + + void advance_to_next_bucket() { // TODO?: refactor to iterator_base class + size_t k = my_index+1; + __TBB_ASSERT( my_bucket, "advancing an invalid iterator?"); + while (k <= my_map->my_mask.load(std::memory_order_relaxed)) { + // Following test uses 2's-complement wizardry + if( k&(k-2) ) // not the beginning of a segment + ++my_bucket; + else my_bucket = my_map->get_bucket( k ); + node_base *n = my_bucket->node_list.load(std::memory_order_relaxed); + if( map_base::is_valid(n) ) { + my_node = static_cast(n); + my_index = k; + return; + } + ++k; + } + my_bucket = nullptr; my_node = nullptr; my_index = k; // the end + } + + template + __TBB_requires(tbb::detail::hash_compare && + ch_map_rw_scoped_lockable) +#else + > + __TBB_requires(tbb::detail::hash_compare) +#endif + friend class concurrent_hash_map; + + hash_map_iterator( const Container &map, std::size_t index, const bucket *b, node_base *n ) : + my_map(&map), my_index(index), my_bucket(b), my_node(nullptr) + { + // Cannot directly initialize to n, because it could be an invalid node pointer (e.g., when + // setting a midpoint for a 1-element range). If it is, try one from a subsequent bucket. + if( map_base::is_valid(n) ) + my_node = static_cast(n); + else if( b ) + advance_to_next_bucket(); + } + + // concurrent_hash_map over which we are iterating. + const Container *my_map; + // Index in hash table for current item + size_t my_index; + // Pointer to bucket + const bucket* my_bucket; + // Pointer to node that has current item + node* my_node; +}; + +template +bool operator==( const hash_map_iterator& i, const hash_map_iterator& j ) { + return i.my_node == j.my_node && i.my_map == j.my_map; +} + +template +bool operator!=( const hash_map_iterator& i, const hash_map_iterator& j ) { + return i.my_node != j.my_node || i.my_map != j.my_map; +} + +// Range class used with concurrent_hash_map +template +class hash_map_range { + using map_type = typename Iterator::map_type; +public: + // Type for size of a range + using size_type = std::size_t; + using value_type = typename Iterator::value_type; + using reference = typename Iterator::reference; + using difference_type = typename Iterator::difference_type; + using iterator = Iterator; + + // True if range is empty. + bool empty() const { return my_begin == my_end; } + + // True if range can be partitioned into two subranges. + bool is_divisible() const { + return my_midpoint != my_end; + } + + // Split range. + hash_map_range( hash_map_range& r, split ) : + my_end(r.my_end), + my_grainsize(r.my_grainsize) + { + r.my_end = my_begin = r.my_midpoint; + __TBB_ASSERT( !empty(), "Splitting despite the range is not divisible" ); + __TBB_ASSERT( !r.empty(), "Splitting despite the range is not divisible" ); + set_midpoint(); + r.set_midpoint(); + } + + // Init range with container and grainsize specified + hash_map_range( const map_type &map, size_type grainsize_ = 1 ) : + my_begin( Iterator( map, 0, map.my_embedded_segment, map.my_embedded_segment->node_list.load(std::memory_order_relaxed) ) ), + my_end( Iterator( map, map.my_mask.load(std::memory_order_relaxed) + 1, nullptr, nullptr ) ), + my_grainsize( grainsize_ ) + { + __TBB_ASSERT( grainsize_>0, "grainsize must be positive" ); + set_midpoint(); + } + + Iterator begin() const { return my_begin; } + Iterator end() const { return my_end; } + // The grain size for this range. + size_type grainsize() const { return my_grainsize; } + +private: + Iterator my_begin; + Iterator my_end; + mutable Iterator my_midpoint; + size_t my_grainsize; + // Set my_midpoint to point approximately half way between my_begin and my_end. + void set_midpoint() const; + template friend class hash_map_range; +}; + +template +void hash_map_range::set_midpoint() const { + // Split by groups of nodes + size_t m = my_end.my_index-my_begin.my_index; + if( m > my_grainsize ) { + m = my_begin.my_index + m/2u; + auto b = my_begin.my_map->get_bucket(m); + my_midpoint = Iterator(*my_begin.my_map,m,b,b->node_list.load(std::memory_order_relaxed)); + } else { + my_midpoint = my_end; + } + __TBB_ASSERT( my_begin.my_index <= my_midpoint.my_index, + "my_begin is after my_midpoint" ); + __TBB_ASSERT( my_midpoint.my_index <= my_end.my_index, + "my_midpoint is after my_end" ); + __TBB_ASSERT( my_begin != my_midpoint || my_begin == my_end, + "[my_begin, my_midpoint) range should not be empty" ); +} + +template , + typename Allocator = tbb_allocator> +#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS + , typename MutexType = spin_rw_mutex + > + __TBB_requires(tbb::detail::hash_compare && + ch_map_rw_scoped_lockable) +#else + > + __TBB_requires(tbb::detail::hash_compare) +#endif +class concurrent_hash_map +#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS + : protected hash_map_base +#else + : protected hash_map_base +#endif +{ + template + friend class hash_map_iterator; + + template + friend class hash_map_range; + using allocator_traits_type = tbb::detail::allocator_traits; + +#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS + using base_type = hash_map_base; +#else + using base_type = hash_map_base; +#endif +public: + using key_type = Key; + using mapped_type = T; + // type_identity is needed to disable implicit deduction guides for std::initializer_list constructors + // and copy/move constructor with explicit allocator argument + using allocator_type = tbb::detail::type_identity_t; + using hash_compare_type = tbb::detail::type_identity_t; + using value_type = std::pair; + using size_type = typename base_type::size_type; + using difference_type = std::ptrdiff_t; +#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS + using mutex_type = MutexType; +#endif + using pointer = typename allocator_traits_type::pointer; + using const_pointer = typename allocator_traits_type::const_pointer; + + using reference = value_type&; + using const_reference = const value_type&; + using iterator = hash_map_iterator; + using const_iterator = hash_map_iterator; + using range_type = hash_map_range; + using const_range_type = hash_map_range; + +protected: + static_assert(std::is_same::value, + "value_type of the container must be the same as its allocator's"); + + friend class const_accessor; + class node; + using segment_index_type = typename base_type::segment_index_type; + using segment_ptr_type = typename base_type::segment_ptr_type; + using node_base = typename base_type::node_base; + using bucket = typename base_type::bucket; + using hashcode_type = typename base_type::hashcode_type; + using bucket_allocator_type = typename base_type::bucket_allocator_type; + using node_allocator_type = typename base_type::allocator_traits_type::template rebind_alloc; + using node_allocator_traits = tbb::detail::allocator_traits; + hash_compare_type my_hash_compare; + + class node : public node_base { + public: + node() {} + ~node() {} + pointer storage() { return &my_value; } + value_type& value() { return *storage(); } + private: + union { + value_type my_value; + }; + }; + + void delete_node( node_base *n ) { + node_allocator_type node_allocator(this->get_allocator()); + node_allocator_traits::destroy(node_allocator, static_cast(n)->storage()); + node_allocator_traits::destroy(node_allocator, static_cast(n)); + node_allocator_traits::deallocate(node_allocator, static_cast(n), 1); + } + + template + static node* create_node(bucket_allocator_type& allocator, Args&&... args) { + node_allocator_type node_allocator(allocator); + node* node_ptr = node_allocator_traits::allocate(node_allocator, 1); + auto guard = make_raii_guard([&] { + node_allocator_traits::destroy(node_allocator, node_ptr); + node_allocator_traits::deallocate(node_allocator, node_ptr, 1); + }); + + node_allocator_traits::construct(node_allocator, node_ptr); + node_allocator_traits::construct(node_allocator, node_ptr->storage(), std::forward(args)...); + guard.dismiss(); + return node_ptr; + } + + static node* allocate_node_copy_construct(bucket_allocator_type& allocator, const Key &key, const T * t){ + return create_node(allocator, key, *t); + } + + static node* allocate_node_move_construct(bucket_allocator_type& allocator, const Key &key, const T * t){ + return create_node(allocator, key, std::move(*const_cast(t))); + } + + template + static node* allocate_node_default_construct(bucket_allocator_type& allocator, const K &key, const T * ){ + // Emplace construct an empty T object inside the pair + return create_node(allocator, std::piecewise_construct, + std::forward_as_tuple(key), std::forward_as_tuple()); + } + + static node* do_not_allocate_node(bucket_allocator_type& , const Key &, const T * ){ + __TBB_ASSERT(false,"this dummy function should not be called"); + return nullptr; + } + + template + node *search_bucket( const K &key, bucket *b ) const { + node *n = static_cast( b->node_list.load(std::memory_order_relaxed) ); + while (this->is_valid(n) && !my_hash_compare.equal(key, n->value().first)) + n = static_cast( n->next ); + __TBB_ASSERT(!rehash_required(n), "Search can be executed only for rehashed bucket"); + return n; + } + + // bucket accessor is to find, rehash, acquire a lock, and access a bucket + class bucket_accessor : public bucket::scoped_type { + bucket *my_b; + public: + bucket_accessor( concurrent_hash_map *base, const hashcode_type h, bool writer = false ) { acquire( base, h, writer ); } + // find a bucket by masked hashcode, optionally rehash, and acquire the lock + inline void acquire( concurrent_hash_map *base, const hashcode_type h, bool writer = false ) { + my_b = base->get_bucket( h ); + // TODO: actually, notification is unnecessary here, just hiding double-check + if (rehash_required(my_b->node_list.load(std::memory_order_acquire)) + && bucket::scoped_type::try_acquire( my_b->mutex, /*write=*/true ) ) + { + if (rehash_required(my_b->node_list.load(std::memory_order_relaxed))) base->rehash_bucket(my_b, h); // recursive rehashing + } + else bucket::scoped_type::acquire( my_b->mutex, writer ); + __TBB_ASSERT(!rehash_required(my_b->node_list.load(std::memory_order_relaxed)), nullptr); + } + + // get bucket pointer + bucket *operator() () { return my_b; } + }; + + // TODO refactor to hash_base + void rehash_bucket( bucket *b_new, const hashcode_type hash ) { + __TBB_ASSERT( hash > 1, "The lowermost buckets can't be rehashed" ); + b_new->node_list.store(reinterpret_cast(empty_rehashed_flag), std::memory_order_release); // mark rehashed + hashcode_type mask = (hashcode_type(1) << tbb::detail::log2(hash)) - 1; // get parent mask from the topmost bit + bucket_accessor b_old( this, hash & mask ); + + mask = (mask<<1) | 1; // get full mask for new bucket + __TBB_ASSERT( (mask&(mask+1))==0 && (hash & mask) == hash, nullptr ); + restart: + node_base* prev = nullptr; + node_base* curr = b_old()->node_list.load(std::memory_order_acquire); + while (this->is_valid(curr)) { + hashcode_type curr_node_hash = my_hash_compare.hash(static_cast(curr)->value().first); + + if ((curr_node_hash & mask) == hash) { + if (!b_old.is_writer()) { + if (!b_old.upgrade_to_writer()) { + goto restart; // node ptr can be invalid due to concurrent erase + } + } + node_base* next = curr->next; + // exclude from b_old + if (prev == nullptr) { + b_old()->node_list.store(curr->next, std::memory_order_relaxed); + } else { + prev->next = curr->next; + } + this->add_to_bucket(b_new, curr); + curr = next; + } else { + prev = curr; + curr = curr->next; + } + } + } + + template + using hash_compare_is_transparent = dependent_bool, U>; + +public: + + class accessor; + // Combines data access, locking, and garbage collection. + class const_accessor : private node::scoped_type /*which derived from no_copy*/ { +#if __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS + friend class concurrent_hash_map; +#else + friend class concurrent_hash_map; +#endif + friend class accessor; + public: + // Type of value + using value_type = const typename concurrent_hash_map::value_type; + + // True if result is empty. + bool empty() const { return !my_node; } + + // Set to null + void release() { + if( my_node ) { + node::scoped_type::release(); + my_node = nullptr; + } + } + + // Return reference to associated value in hash table. + const_reference operator*() const { + __TBB_ASSERT( my_node, "attempt to dereference empty accessor" ); + return my_node->value(); + } + + // Return pointer to associated value in hash table. + const_pointer operator->() const { + return &operator*(); + } + + // Create empty result + const_accessor() : my_node(nullptr), my_hash() {} + + // Destroy result after releasing the underlying reference. + ~const_accessor() { + my_node = nullptr; // scoped lock's release() is called in its destructor + } + protected: + bool is_writer() { return node::scoped_type::is_writer(); } + node *my_node; + hashcode_type my_hash; + }; + + // Allows write access to elements and combines data access, locking, and garbage collection. + class accessor: public const_accessor { + public: + // Type of value + using value_type = typename concurrent_hash_map::value_type; + + // Return reference to associated value in hash table. + reference operator*() const { + __TBB_ASSERT( this->my_node, "attempt to dereference empty accessor" ); + return this->my_node->value(); + } + + // Return pointer to associated value in hash table. + pointer operator->() const { + return &operator*(); + } + }; + + explicit concurrent_hash_map( const hash_compare_type& compare, const allocator_type& a = allocator_type() ) + : base_type(a) + , my_hash_compare(compare) + {} + + concurrent_hash_map() : concurrent_hash_map(hash_compare_type()) {} + + explicit concurrent_hash_map( const allocator_type& a ) + : concurrent_hash_map(hash_compare_type(), a) + {} + + // Construct empty table with n preallocated buckets. This number serves also as initial concurrency level. + concurrent_hash_map( size_type n, const allocator_type &a = allocator_type() ) + : concurrent_hash_map(a) + { + this->reserve(n); + } + + concurrent_hash_map( size_type n, const hash_compare_type& compare, const allocator_type& a = allocator_type() ) + : concurrent_hash_map(compare, a) + { + this->reserve(n); + } + + // Copy constructor + concurrent_hash_map( const concurrent_hash_map &table ) + : concurrent_hash_map(node_allocator_traits::select_on_container_copy_construction(table.get_allocator())) + { + try_call( [&] { + internal_copy(table); + }).on_exception( [&] { + this->clear(); + }); + } + + concurrent_hash_map( const concurrent_hash_map &table, const allocator_type &a) + : concurrent_hash_map(a) + { + try_call( [&] { + internal_copy(table); + }).on_exception( [&] { + this->clear(); + }); + } + + // Move constructor + concurrent_hash_map( concurrent_hash_map &&table ) + : concurrent_hash_map(std::move(table.get_allocator())) + { + this->internal_move(std::move(table)); + } + + // Move constructor + concurrent_hash_map( concurrent_hash_map &&table, const allocator_type &a ) + : concurrent_hash_map(a) + { + using is_equal_type = typename node_allocator_traits::is_always_equal; + internal_move_construct_with_allocator(std::move(table), a, is_equal_type()); + } + + // Construction with copying iteration range and given allocator instance + template + concurrent_hash_map( I first, I last, const allocator_type &a = allocator_type() ) + : concurrent_hash_map(a) + { + try_call( [&] { + internal_copy(first, last, std::distance(first, last)); + }).on_exception( [&] { + this->clear(); + }); + } + + template + concurrent_hash_map( I first, I last, const hash_compare_type& compare, const allocator_type& a = allocator_type() ) + : concurrent_hash_map(compare, a) + { + try_call( [&] { + internal_copy(first, last, std::distance(first, last)); + }).on_exception( [&] { + this->clear(); + }); + } + + concurrent_hash_map( std::initializer_list il, const hash_compare_type& compare = hash_compare_type(), const allocator_type& a = allocator_type() ) + : concurrent_hash_map(compare, a) + { + try_call( [&] { + internal_copy(il.begin(), il.end(), il.size()); + }).on_exception( [&] { + this->clear(); + }); + } + + concurrent_hash_map( std::initializer_list il, const allocator_type& a ) + : concurrent_hash_map(il, hash_compare_type(), a) {} + + // Assignment + concurrent_hash_map& operator=( const concurrent_hash_map &table ) { + if( this != &table ) { + clear(); + copy_assign_allocators(this->my_allocator, table.my_allocator); + internal_copy(table); + } + return *this; + } + + // Move Assignment + concurrent_hash_map& operator=( concurrent_hash_map &&table ) { + if( this != &table ) { + using pocma_type = typename node_allocator_traits::propagate_on_container_move_assignment; + using is_equal_type = typename node_allocator_traits::is_always_equal; + move_assign_allocators(this->my_allocator, table.my_allocator); + internal_move_assign(std::move(table), tbb::detail::disjunction()); + } + return *this; + } + + // Assignment + concurrent_hash_map& operator=( std::initializer_list il ) { + clear(); + internal_copy(il.begin(), il.end(), il.size()); + return *this; + } + + // Rehashes and optionally resizes the whole table. + /** Useful to optimize performance before or after concurrent operations. + Also enables using of find() and count() concurrent methods in serial context. */ + void rehash(size_type sz = 0) { + this->reserve(sz); // TODO: add reduction of number of buckets as well + hashcode_type mask = this->my_mask.load(std::memory_order_relaxed); + hashcode_type b = (mask+1)>>1; // size or first index of the last segment + __TBB_ASSERT((b&(b-1))==0, nullptr); // zero or power of 2 + bucket *bp = this->get_bucket( b ); // only the last segment should be scanned for rehashing + for(; b <= mask; b++, bp++ ) { + node_base *n = bp->node_list.load(std::memory_order_relaxed); + __TBB_ASSERT( this->is_valid(n) || empty_rehashed(n) || rehash_required(n), "Broken internal structure" ); + __TBB_ASSERT( *reinterpret_cast(&bp->mutex) == 0, "concurrent or unexpectedly terminated operation during rehash() execution" ); + if (rehash_required(n)) { // rehash bucket, conditional because rehashing of a previous bucket may affect this one + hashcode_type h = b; bucket *b_old = bp; + do { + __TBB_ASSERT( h > 1, "The lowermost buckets can't be rehashed" ); + hashcode_type m = ( hashcode_type(1) << tbb::detail::log2( h ) ) - 1; // get parent mask from the topmost bit + b_old = this->get_bucket( h &= m ); + } while( rehash_required(b_old->node_list.load(std::memory_order_relaxed)) ); + // now h - is index of the root rehashed bucket b_old + this->mark_rehashed_levels( h ); // mark all non-rehashed children recursively across all segments + node_base* prev = nullptr; + node_base* curr = b_old->node_list.load(std::memory_order_relaxed); + while (this->is_valid(curr)) { + hashcode_type curr_node_hash = my_hash_compare.hash(static_cast(curr)->value().first); + + if ((curr_node_hash & mask) != h) { // should be rehashed + node_base* next = curr->next; + // exclude from b_old + if (prev == nullptr) { + b_old->node_list.store(curr->next, std::memory_order_relaxed); + } else { + prev->next = curr->next; + } + bucket *b_new = this->get_bucket(curr_node_hash & mask); + __TBB_ASSERT(!rehash_required(b_new->node_list.load(std::memory_order_relaxed)), "hash() function changed for key in table or internal error"); + this->add_to_bucket(b_new, curr); + curr = next; + } else { + prev = curr; + curr = curr->next; + } + } + } + } + } + + // Clear table + void clear() { + hashcode_type m = this->my_mask.load(std::memory_order_relaxed); + __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); + this->my_size.store(0, std::memory_order_relaxed); + segment_index_type s = this->segment_index_of( m ) + 1; + __TBB_ASSERT( s == this->pointers_per_table || !this->my_table[s].load(std::memory_order_relaxed), "wrong mask or concurrent grow" ); + while(s != 0) { + s--; + __TBB_ASSERT(this->is_valid(this->my_table[s].load(std::memory_order_relaxed)), "wrong mask or concurrent grow" ); + segment_ptr_type buckets_ptr = this->my_table[s].load(std::memory_order_relaxed); + size_type sz = this->segment_size( s ? s : 1 ); + for( segment_index_type i = 0; i < sz; i++ ) + for( node_base *n = buckets_ptr[i].node_list.load(std::memory_order_relaxed); + this->is_valid(n); n = buckets_ptr[i].node_list.load(std::memory_order_relaxed) ) + { + buckets_ptr[i].node_list.store(n->next, std::memory_order_relaxed); + delete_node( n ); + } + this->delete_segment(s); + } + this->my_mask.store(this->embedded_buckets - 1, std::memory_order_relaxed); + } + + // Clear table and destroy it. + ~concurrent_hash_map() { clear(); } + + //------------------------------------------------------------------------ + // Parallel algorithm support + //------------------------------------------------------------------------ + range_type range( size_type grainsize=1 ) { + return range_type( *this, grainsize ); + } + const_range_type range( size_type grainsize=1 ) const { + return const_range_type( *this, grainsize ); + } + + //------------------------------------------------------------------------ + // STL support - not thread-safe methods + //------------------------------------------------------------------------ + iterator begin() { return iterator( *this, 0, this->my_embedded_segment, this->my_embedded_segment->node_list.load(std::memory_order_relaxed) ); } + const_iterator begin() const { return const_iterator( *this, 0, this->my_embedded_segment, this->my_embedded_segment->node_list.load(std::memory_order_relaxed) ); } + const_iterator cbegin() const { return const_iterator( *this, 0, this->my_embedded_segment, this->my_embedded_segment->node_list.load(std::memory_order_relaxed) ); } + iterator end() { return iterator( *this, 0, nullptr, nullptr ); } + const_iterator end() const { return const_iterator( *this, 0, nullptr, nullptr ); } + const_iterator cend() const { return const_iterator( *this, 0, nullptr, nullptr ); } + std::pair equal_range( const Key& key ) { return internal_equal_range( key, end() ); } + std::pair equal_range( const Key& key ) const { return internal_equal_range( key, end() ); } + + template + typename std::enable_if::value, + std::pair>::type equal_range( const K& key ) { + return internal_equal_range(key, end()); + } + + template + typename std::enable_if::value, + std::pair>::type equal_range( const K& key ) const { + return internal_equal_range(key, end()); + } + + // Number of items in table. + size_type size() const { return this->my_size.load(std::memory_order_acquire); } + + // True if size()==0. + __TBB_nodiscard bool empty() const { return size() == 0; } + + // Upper bound on size. + size_type max_size() const { + return allocator_traits_type::max_size(base_type::get_allocator()); + } + + // Returns the current number of buckets + size_type bucket_count() const { return this->my_mask.load(std::memory_order_relaxed) + 1; } + + // return allocator object + allocator_type get_allocator() const { return base_type::get_allocator(); } + + // swap two instances. Iterators are invalidated + void swap(concurrent_hash_map& table) { + using pocs_type = typename node_allocator_traits::propagate_on_container_swap; + using is_equal_type = typename node_allocator_traits::is_always_equal; + swap_allocators(this->my_allocator, table.my_allocator); + internal_swap(table, tbb::detail::disjunction()); + } + + //------------------------------------------------------------------------ + // concurrent map operations + //------------------------------------------------------------------------ + + // Return count of items (0 or 1) + size_type count( const Key &key ) const { + return const_cast(this)->lookup(key, nullptr, nullptr, /*write=*/false, &do_not_allocate_node); + } + + template + typename std::enable_if::value, + size_type>::type count( const K& key ) const { + return const_cast(this)->lookup(key, nullptr, nullptr, /*write=*/false, &do_not_allocate_node); + } + + // Find item and acquire a read lock on the item. + /** Return true if item is found, false otherwise. */ + bool find( const_accessor &result, const Key &key ) const { + result.release(); + return const_cast(this)->lookup(key, nullptr, &result, /*write=*/false, &do_not_allocate_node ); + } + + // Find item and acquire a write lock on the item. + /** Return true if item is found, false otherwise. */ + bool find( accessor &result, const Key &key ) { + result.release(); + return lookup(key, nullptr, &result, /*write=*/true, &do_not_allocate_node); + } + + template + typename std::enable_if::value, + bool>::type find( const_accessor& result, const K& key ) { + result.release(); + return lookup(key, nullptr, &result, /*write=*/false, &do_not_allocate_node); + } + + template + typename std::enable_if::value, + bool>::type find( accessor& result, const K& key ) { + result.release(); + return lookup(key, nullptr, &result, /*write=*/true, &do_not_allocate_node); + } + + // Insert item (if not already present) and acquire a read lock on the item. + /** Returns true if item is new. */ + bool insert( const_accessor &result, const Key &key ) { + result.release(); + return lookup(key, nullptr, &result, /*write=*/false, &allocate_node_default_construct<>); + } + + // Insert item (if not already present) and acquire a write lock on the item. + /** Returns true if item is new. */ + bool insert( accessor &result, const Key &key ) { + result.release(); + return lookup(key, nullptr, &result, /*write=*/true, &allocate_node_default_construct<>); + } + + template + typename std::enable_if::value && + std::is_constructible::value, + bool>::type insert( const_accessor& result, const K& key ) { + result.release(); + return lookup(key, nullptr, &result, /*write=*/false, &allocate_node_default_construct); + } + + template + typename std::enable_if::value && + std::is_constructible::value, + bool>::type insert( accessor& result, const K& key ) { + result.release(); + return lookup(key, nullptr, &result, /*write=*/true, &allocate_node_default_construct); + } + + // Insert item by copying if there is no such key present already and acquire a read lock on the item. + /** Returns true if item is new. */ + bool insert( const_accessor &result, const value_type &value ) { + result.release(); + return lookup(value.first, &value.second, &result, /*write=*/false, &allocate_node_copy_construct); + } + + // Insert item by copying if there is no such key present already and acquire a write lock on the item. + /** Returns true if item is new. */ + bool insert( accessor &result, const value_type &value ) { + result.release(); + return lookup(value.first, &value.second, &result, /*write=*/true, &allocate_node_copy_construct); + } + + // Insert item by copying if there is no such key present already + /** Returns true if item is inserted. */ + bool insert( const value_type &value ) { + return lookup(value.first, &value.second, nullptr, /*write=*/false, &allocate_node_copy_construct); + } + + // Insert item by copying if there is no such key present already and acquire a read lock on the item. + /** Returns true if item is new. */ + bool insert( const_accessor &result, value_type && value ) { + return generic_move_insert(result, std::move(value)); + } + + // Insert item by copying if there is no such key present already and acquire a write lock on the item. + /** Returns true if item is new. */ + bool insert( accessor &result, value_type && value ) { + return generic_move_insert(result, std::move(value)); + } + + // Insert item by copying if there is no such key present already + /** Returns true if item is inserted. */ + bool insert( value_type && value ) { + return generic_move_insert(accessor_not_used(), std::move(value)); + } + + // Insert item by copying if there is no such key present already and acquire a read lock on the item. + /** Returns true if item is new. */ + template + bool emplace( const_accessor &result, Args&&... args ) { + return generic_emplace(result, std::forward(args)...); + } + + // Insert item by copying if there is no such key present already and acquire a write lock on the item. + /** Returns true if item is new. */ + template + bool emplace( accessor &result, Args&&... args ) { + return generic_emplace(result, std::forward(args)...); + } + + // Insert item by copying if there is no such key present already + /** Returns true if item is inserted. */ + template + bool emplace( Args&&... args ) { + return generic_emplace(accessor_not_used(), std::forward(args)...); + } + + // Insert range [first, last) + template + void insert( I first, I last ) { + for ( ; first != last; ++first ) + insert( *first ); + } + + // Insert initializer list + void insert( std::initializer_list il ) { + insert( il.begin(), il.end() ); + } + + // Erase item. + /** Return true if item was erased by particularly this call. */ + bool erase( const Key &key ) { + return internal_erase(key); + } + + template + typename std::enable_if::value, + bool>::type erase( const K& key ) { + return internal_erase(key); + } + + // Erase item by const_accessor. + /** Return true if item was erased by particularly this call. */ + bool erase( const_accessor& item_accessor ) { + return exclude( item_accessor ); + } + + // Erase item by accessor. + /** Return true if item was erased by particularly this call. */ + bool erase( accessor& item_accessor ) { + return exclude( item_accessor ); + } + +protected: + template + node* allocate_node_helper( const K& key, const T* t, AllocateNodeType allocate_node, std::true_type ) { + return allocate_node(base_type::get_allocator(), key, t); + } + + template + node* allocate_node_helper( const K&, const T*, AllocateNodeType, std::false_type ) { + __TBB_ASSERT(false, "allocate_node_helper with std::false_type should never been called"); + return nullptr; + } + + // Insert or find item and optionally acquire a lock on the item. + template + bool lookup( const K &key, const T *t, const_accessor *result, bool write, AllocateNodeType allocate_node, node *tmp_n = nullptr) + { + __TBB_ASSERT( !result || !result->my_node, nullptr ); + bool return_value; + hashcode_type const h = my_hash_compare.hash( key ); + hashcode_type m = this->my_mask.load(std::memory_order_acquire); + segment_index_type grow_segment = 0; + node *n; + restart: + {//lock scope + __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); + return_value = false; + // get bucket + bucket_accessor b( this, h & m ); + // find a node + n = search_bucket( key, b() ); + if( OpInsert ) { + // [opt] insert a key + if( !n ) { + if( !tmp_n ) { + tmp_n = allocate_node_helper(key, t, allocate_node, std::integral_constant{}); + } + while ( !b.is_writer() && !b.upgrade_to_writer() ) { // TODO: improved insertion + // Rerun search list, in case another thread inserted the intem during the upgrade + n = search_bucket(key, b()); + if (this->is_valid(n)) { // unfortunately, it did + if (!b.downgrade_to_reader()) { + // If the lock was downgraded with reacquiring the mutex + // Rerun search list in case another thread removed the item during the downgrade + n = search_bucket(key, b()); + if (!this->is_valid(n)) { + // Unfortunately, it did + // We need to try upgrading to writer again + continue; + } + } + goto exists; + } + } + + if( this->check_mask_race(h, m) ) + goto restart; // b.release() is done in ~b(). + // insert and set flag to grow the container + grow_segment = this->insert_new_node( b(), n = tmp_n, m ); + tmp_n = nullptr; + return_value = true; + } + } else { // find or count + if( !n ) { + if( this->check_mask_race( h, m ) ) + goto restart; // b.release() is done in ~b(). TODO: replace by continue + return false; + } + return_value = true; + } + exists: + if( !result ) goto check_growth; + // TODO: the following seems as generic/regular operation + // acquire the item + if( !result->try_acquire( n->mutex, write ) ) { + for( tbb::detail::atomic_backoff backoff(true);; ) { + if( result->try_acquire( n->mutex, write ) ) break; + if( !backoff.bounded_pause() ) { + // the wait takes really long, restart the operation + b.release(); + __TBB_ASSERT( !OpInsert || !return_value, "Can't acquire new item in locked bucket?" ); + yield(); + m = this->my_mask.load(std::memory_order_acquire); + goto restart; + } + } + } + }//lock scope + result->my_node = n; + result->my_hash = h; + check_growth: + // [opt] grow the container + if( grow_segment ) { + this->enable_segment( grow_segment ); + } + if( tmp_n ) // if OpInsert only + delete_node( tmp_n ); + return return_value; + } + + struct accessor_not_used { void release(){}}; + friend const_accessor* accessor_location( accessor_not_used const& ){ return nullptr;} + friend const_accessor* accessor_location( const_accessor & a ) { return &a;} + + friend bool is_write_access_needed( accessor const& ) { return true;} + friend bool is_write_access_needed( const_accessor const& ) { return false;} + friend bool is_write_access_needed( accessor_not_used const& ) { return false;} + + template + bool generic_move_insert( Accessor && result, value_type && value ) { + result.release(); + return lookup(value.first, &value.second, accessor_location(result), is_write_access_needed(result), &allocate_node_move_construct); + } + + template + bool generic_emplace( Accessor && result, Args &&... args ) { + result.release(); + node * node_ptr = create_node(base_type::get_allocator(), std::forward(args)...); + return lookup(node_ptr->value().first, nullptr, accessor_location(result), is_write_access_needed(result), &do_not_allocate_node, node_ptr); + } + + // delete item by accessor + bool exclude( const_accessor &item_accessor ) { + __TBB_ASSERT( item_accessor.my_node, nullptr ); + node_base *const exclude_node = item_accessor.my_node; + hashcode_type const hash = item_accessor.my_hash; + hashcode_type mask = this->my_mask.load(std::memory_order_acquire); + do { + // get bucket + bucket_accessor b( this, hash & mask, /*writer=*/true ); + node_base* prev = nullptr; + node_base* curr = b()->node_list.load(std::memory_order_relaxed); + + while (curr && curr != exclude_node) { + prev = curr; + curr = curr->next; + } + + if (curr == nullptr) { // someone else was first + if (this->check_mask_race(hash, mask)) + continue; + item_accessor.release(); + return false; + } + __TBB_ASSERT( curr == exclude_node, nullptr ); + // remove from container + if (prev == nullptr) { + b()->node_list.store(curr->next, std::memory_order_relaxed); + } else { + prev->next = curr->next; + } + + this->my_size--; + break; + } while(true); + if (!item_accessor.is_writer()) { // need to get exclusive lock + item_accessor.upgrade_to_writer(); // return value means nothing here + } + + item_accessor.release(); + delete_node(exclude_node); // Only one thread can delete it + return true; + } + + template + bool internal_erase( const K& key ) { + node_base *erase_node; + hashcode_type const hash = my_hash_compare.hash(key); + hashcode_type mask = this->my_mask.load(std::memory_order_acquire); + restart: + {//lock scope + // get bucket + bucket_accessor b( this, hash & mask ); + search: + node_base* prev = nullptr; + erase_node = b()->node_list.load(std::memory_order_relaxed); + while (this->is_valid(erase_node) && !my_hash_compare.equal(key, static_cast(erase_node)->value().first ) ) { + prev = erase_node; + erase_node = erase_node->next; + } + + if (erase_node == nullptr) { // not found, but mask could be changed + if (this->check_mask_race(hash, mask)) + goto restart; + return false; + } else if (!b.is_writer() && !b.upgrade_to_writer()) { + if (this->check_mask_race(hash, mask)) // contended upgrade, check mask + goto restart; + goto search; + } + + // remove from container + if (prev == nullptr) { + b()->node_list.store(erase_node->next, std::memory_order_relaxed); + } else { + prev->next = erase_node->next; + } + this->my_size--; + } + { + typename node::scoped_type item_locker( erase_node->mutex, /*write=*/true ); + } + // note: there should be no threads pretending to acquire this mutex again, do not try to upgrade const_accessor! + delete_node(erase_node); // Only one thread can delete it due to write lock on the bucket + return true; + } + + // Returns an iterator for an item defined by the key, or for the next item after it (if upper==true) + template + std::pair internal_equal_range( const K& key, I end_ ) const { + hashcode_type h = my_hash_compare.hash( key ); + hashcode_type m = this->my_mask.load(std::memory_order_relaxed); + __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); + h &= m; + bucket *b = this->get_bucket( h ); + while (rehash_required(b->node_list.load(std::memory_order_relaxed))) { + m = ( hashcode_type(1) << tbb::detail::log2( h ) ) - 1; // get parent mask from the topmost bit + b = this->get_bucket( h &= m ); + } + node *n = search_bucket( key, b ); + if( !n ) + return std::make_pair(end_, end_); + iterator lower(*this, h, b, n), upper(lower); + return std::make_pair(lower, ++upper); + } + + // Copy "source" to *this, where *this must start out empty. + void internal_copy( const concurrent_hash_map& source ) { + hashcode_type mask = source.my_mask.load(std::memory_order_relaxed); + if( this->my_mask.load(std::memory_order_relaxed) == mask ) { // optimized version + this->reserve(source.my_size.load(std::memory_order_relaxed)); // TODO: load_factor? + bucket *dst = nullptr, *src = nullptr; + bool rehashing_required = false; + for( hashcode_type k = 0; k <= mask; k++ ) { + if( k & (k-2) ) ++dst,src++; // not the beginning of a segment + else { dst = this->get_bucket( k ); src = source.get_bucket( k ); } + __TBB_ASSERT(!rehash_required(dst->node_list.load(std::memory_order_relaxed)), "Invalid bucket in destination table"); + node *n = static_cast( src->node_list.load(std::memory_order_relaxed) ); + if (rehash_required(n)) { // source is not rehashed, items are in previous buckets + rehashing_required = true; + dst->node_list.store(reinterpret_cast(rehash_req_flag), std::memory_order_relaxed); + } else for(; n; n = static_cast( n->next ) ) { + node* node_ptr = create_node(base_type::get_allocator(), n->value().first, n->value().second); + this->add_to_bucket( dst, node_ptr); + this->my_size.fetch_add(1, std::memory_order_relaxed); + } + } + if( rehashing_required ) rehash(); + } else internal_copy(source.begin(), source.end(), source.my_size.load(std::memory_order_relaxed)); + } + + template + void internal_copy( I first, I last, size_type reserve_size ) { + this->reserve(reserve_size); // TODO: load_factor? + hashcode_type m = this->my_mask.load(std::memory_order_relaxed); + for(; first != last; ++first) { + const auto& key = (*first).first; + hashcode_type h = my_hash_compare.hash(key); + bucket *b = this->get_bucket( h & m ); + __TBB_ASSERT(!rehash_required(b->node_list.load(std::memory_order_relaxed)), "Invalid bucket in destination table"); + + if (search_bucket(key, b) == nullptr) { + node* node_ptr = create_node(base_type::get_allocator(), *first); + this->add_to_bucket( b, node_ptr ); + ++this->my_size; // TODO: replace by non-atomic op + } + } + } + + void internal_move_construct_with_allocator( concurrent_hash_map&& other, const allocator_type&, + /*is_always_equal=*/std::true_type ) + { + this->internal_move(std::move(other)); + } + + void internal_move_construct_with_allocator( concurrent_hash_map&& other, const allocator_type& a, + /*is_always_equal=*/std::false_type ) + { + if (a == other.get_allocator()){ + this->internal_move(std::move(other)); + } else { + try_call( [&] { + internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end()), + other.size()); + }).on_exception( [&] { + this->clear(); + }); + } + } + + void internal_move_assign( concurrent_hash_map&& other, + /*is_always_equal || POCMA = */std::true_type) + { + this->internal_move(std::move(other)); + } + + void internal_move_assign(concurrent_hash_map&& other, /*is_always_equal=*/ std::false_type) { + if (this->my_allocator == other.my_allocator) { + this->internal_move(std::move(other)); + } else { + //do per element move + internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end()), + other.size()); + } + } + + void internal_swap(concurrent_hash_map& other, /*is_always_equal || POCS = */ std::true_type) { + this->internal_swap_content(other); + } + + void internal_swap(concurrent_hash_map& other, /*is_always_equal || POCS = */ std::false_type) { + __TBB_ASSERT(this->my_allocator == other.my_allocator, nullptr); + this->internal_swap_content(other); + } + + // Fast find when no concurrent erasure is used. For internal use inside TBB only! + /** Return pointer to item with given key, or nullptr if no such item exists. + Must not be called concurrently with erasure operations. */ + const_pointer internal_fast_find( const Key& key ) const { + hashcode_type h = my_hash_compare.hash( key ); + hashcode_type m = this->my_mask.load(std::memory_order_acquire); + node *n; + restart: + __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); + bucket *b = this->get_bucket( h & m ); + // TODO: actually, notification is unnecessary here, just hiding double-check + if (rehash_required(b->node_list.load(std::memory_order_acquire))) + { + typename bucket::scoped_type lock; + if( lock.try_acquire( b->mutex, /*write=*/true ) ) { + if (rehash_required(b->node_list.load(std::memory_order_relaxed))) + const_cast(this)->rehash_bucket( b, h & m ); //recursive rehashing + } + else lock.acquire( b->mutex, /*write=*/false ); + __TBB_ASSERT(!rehash_required(b->node_list.load(std::memory_order_relaxed)), nullptr); + } + n = search_bucket( key, b ); + if( n ) + return n->storage(); + else if( this->check_mask_race( h, m ) ) + goto restart; + return nullptr; + } +}; + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +template >, + typename Alloc = tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_hash_map( It, It, HashCompare = HashCompare(), Alloc = Alloc() ) +-> concurrent_hash_map, iterator_mapped_t, HashCompare, Alloc>; + +template >, + typename = std::enable_if_t>> +concurrent_hash_map( It, It, Alloc ) +-> concurrent_hash_map, iterator_mapped_t, d1::tbb_hash_compare>, Alloc>; + +template >, + typename Alloc = tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_hash_map( std::initializer_list>, HashCompare = HashCompare(), Alloc = Alloc() ) +-> concurrent_hash_map, T, HashCompare, Alloc>; + +template >> +concurrent_hash_map( std::initializer_list>, Alloc ) +-> concurrent_hash_map, T, d1::tbb_hash_compare>, Alloc>; + +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + +template +inline bool operator==(const concurrent_hash_map &a, const concurrent_hash_map &b) { + if(a.size() != b.size()) return false; + typename concurrent_hash_map::const_iterator i(a.begin()), i_end(a.end()); + typename concurrent_hash_map::const_iterator j, j_end(b.end()); + for(; i != i_end; ++i) { + j = b.equal_range(i->first).first; + if( j == j_end || !(i->second == j->second) ) return false; + } + return true; +} + +#if !__TBB_CPP20_COMPARISONS_PRESENT +template +inline bool operator!=(const concurrent_hash_map &a, const concurrent_hash_map &b) +{ return !(a == b); } +#endif // !__TBB_CPP20_COMPARISONS_PRESENT + +template +inline void swap(concurrent_hash_map &a, concurrent_hash_map &b) +{ a.swap( b ); } + +} // namespace d2 +} // namespace detail + +inline namespace v1 { + using detail::split; + using detail::d2::concurrent_hash_map; + using detail::d1::tbb_hash_compare; +} // namespace v1 + +} // namespace tbb + +#endif /* __TBB_concurrent_hash_map_H */ diff --git a/src/3rd party/oneapi/tbb/concurrent_lru_cache.h b/src/3rd party/oneapi/tbb/concurrent_lru_cache.h new file mode 100644 index 0000000000..83d0576eae --- /dev/null +++ b/src/3rd party/oneapi/tbb/concurrent_lru_cache.h @@ -0,0 +1,374 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_lru_cache_H +#define __TBB_concurrent_lru_cache_H + +#if ! TBB_PREVIEW_CONCURRENT_LRU_CACHE + #error Set TBB_PREVIEW_CONCURRENT_LRU_CACHE to include concurrent_lru_cache.h +#endif + +#include "detail/_assert.h" +#include "detail/_aggregator.h" + +#include // for std::map +#include // for std::list +#include // for std::make_pair +#include // for std::find +#include // for std::atomic + +namespace tbb { + +namespace detail { +namespace d1 { + +//----------------------------------------------------------------------------- +// Concurrent LRU cache +//----------------------------------------------------------------------------- + +template +class concurrent_lru_cache : no_assign { +// incapsulated helper classes +private: + struct handle_object; + struct storage_map_value_type; + + struct aggregator_operation; + struct retrieve_aggregator_operation; + struct signal_end_of_usage_aggregator_operation; + +// typedefs +public: + using key_type = KeyT; + using value_type = ValT; + using pointer = ValT*; + using reference = ValT&; + using const_pointer = const ValT*; + using const_reference = const ValT&; + + using value_function_type = KeyToValFunctorT; + using handle = handle_object; +private: + using lru_cache_type = concurrent_lru_cache; + + using storage_map_type = std::map; + using storage_map_iterator_type = typename storage_map_type::iterator; + using storage_map_pointer_type = typename storage_map_type::pointer; + using storage_map_reference_type = typename storage_map_type::reference; + + using history_list_type = std::list; + using history_list_iterator_type = typename history_list_type::iterator; + + using aggregator_operation_type = aggregator_operation; + using aggregator_function_type = aggregating_functor; + using aggregator_type = aggregator; + + friend class aggregating_functor; + +// fields +private: + value_function_type my_value_function; + aggregator_type my_aggregator; + + storage_map_type my_storage_map; // storage map for used objects + history_list_type my_history_list; // history list for unused objects + const std::size_t my_history_list_capacity; // history list's allowed capacity + +// interface +public: + + concurrent_lru_cache(value_function_type value_function, std::size_t cache_capacity) + : my_value_function(value_function), my_history_list_capacity(cache_capacity) { + my_aggregator.initialize_handler(aggregator_function_type(this)); + } + + handle operator[](key_type key) { + retrieve_aggregator_operation op(key); + my_aggregator.execute(&op); + + if (op.is_new_value_needed()) { + op.result().second.my_value = my_value_function(key); + op.result().second.my_is_ready.store(true, std::memory_order_release); + } else { + spin_wait_while_eq(op.result().second.my_is_ready, false); + } + + return handle(*this, op.result()); + } + +private: + + void handle_operations(aggregator_operation* op_list) { + while (op_list) { + op_list->cast_and_handle(*this); + aggregator_operation* prev_op = op_list; + op_list = op_list->next; + + (prev_op->status).store(1, std::memory_order_release); + } + } + + void signal_end_of_usage(storage_map_reference_type map_record_ref) { + signal_end_of_usage_aggregator_operation op(map_record_ref); + my_aggregator.execute(&op); + } + + void signal_end_of_usage_serial(storage_map_reference_type map_record_ref) { + storage_map_iterator_type map_it = my_storage_map.find(map_record_ref.first); + + __TBB_ASSERT(map_it != my_storage_map.end(), + "cache should not return past-end iterators to outer world"); + __TBB_ASSERT(&(*map_it) == &map_record_ref, + "dangling reference has been returned to outside world: data race?"); + __TBB_ASSERT(std::find(my_history_list.begin(), my_history_list.end(), map_it) == my_history_list.end(), + "object in use should not be in list of unused objects "); + + // if it was the last reference, put it to the LRU history + if (! --(map_it->second.my_ref_counter)) { + // if the LRU history is full, evict the oldest items to get space + if (my_history_list.size() >= my_history_list_capacity) { + if (my_history_list_capacity == 0) { + // Since LRU history capacity is zero, there is no need to keep the element in history + my_storage_map.erase(map_it); + return; + } + std::size_t number_of_elements_to_evict = 1 + my_history_list.size() - my_history_list_capacity; + + for (std::size_t i = 0; i < number_of_elements_to_evict; ++i) { + storage_map_iterator_type map_it_to_evict = my_history_list.back(); + + __TBB_ASSERT(map_it_to_evict->second.my_ref_counter == 0, + "item to be evicted should not have a live references"); + + // TODO: can we use forward_list instead of list? pop_front / insert_after last + my_history_list.pop_back(); + my_storage_map.erase(map_it_to_evict); + } + } + + // TODO: can we use forward_list instead of list? pop_front / insert_after last + my_history_list.push_front(map_it); + map_it->second.my_history_list_iterator = my_history_list.begin(); + } + } + + storage_map_reference_type retrieve_serial(key_type key, bool& is_new_value_needed) { + storage_map_iterator_type map_it = my_storage_map.find(key); + + if (map_it == my_storage_map.end()) { + map_it = my_storage_map.emplace_hint( + map_it, std::piecewise_construct, std::make_tuple(key), std::make_tuple(value_type(), 0, my_history_list.end(), false)); + is_new_value_needed = true; + } else { + history_list_iterator_type list_it = map_it->second.my_history_list_iterator; + if (list_it != my_history_list.end()) { + __TBB_ASSERT(map_it->second.my_ref_counter == 0, + "item to be evicted should not have a live references"); + + // Item is going to be used. Therefore it is not a subject for eviction, + // so we remove it from LRU history. + my_history_list.erase(list_it); + map_it->second.my_history_list_iterator = my_history_list.end(); + } + } + + ++(map_it->second.my_ref_counter); + return *map_it; + } +}; + +//----------------------------------------------------------------------------- +// Value type for storage map in concurrent LRU cache +//----------------------------------------------------------------------------- + +template +struct concurrent_lru_cache::storage_map_value_type { +//typedefs +public: + using ref_counter_type = std::size_t; + +// fields +public: + value_type my_value; + ref_counter_type my_ref_counter; + history_list_iterator_type my_history_list_iterator; + std::atomic my_is_ready; + +// interface +public: + storage_map_value_type( + value_type const& value, ref_counter_type ref_counter, + history_list_iterator_type history_list_iterator, bool is_ready) + : my_value(value), my_ref_counter(ref_counter), + my_history_list_iterator(history_list_iterator), my_is_ready(is_ready) {} +}; + +//----------------------------------------------------------------------------- +// Handle object for operator[] in concurrent LRU cache +//----------------------------------------------------------------------------- + +template +struct concurrent_lru_cache::handle_object { +// fields +private: + lru_cache_type* my_lru_cache_ptr; + storage_map_pointer_type my_map_record_ptr; + +// interface +public: + handle_object() + : my_lru_cache_ptr(nullptr), my_map_record_ptr(nullptr) {} + handle_object(lru_cache_type& lru_cache_ref, storage_map_reference_type map_record_ref) + : my_lru_cache_ptr(&lru_cache_ref), my_map_record_ptr(&map_record_ref) {} + + handle_object(handle_object&) = delete; + void operator=(handle_object&) = delete; + + handle_object(handle_object&& other) + : my_lru_cache_ptr(other.my_lru_cache_ptr), my_map_record_ptr(other.my_map_record_ptr) { + + __TBB_ASSERT( + (other.my_lru_cache_ptr != nullptr && other.my_map_record_ptr != nullptr) || + (other.my_lru_cache_ptr == nullptr && other.my_map_record_ptr == nullptr), + "invalid state of moving object?"); + + other.my_lru_cache_ptr = nullptr; + other.my_map_record_ptr = nullptr; + } + + handle_object& operator=(handle_object&& other) { + __TBB_ASSERT( + (other.my_lru_cache_ptr != nullptr && other.my_map_record_ptr != nullptr) || + (other.my_lru_cache_ptr == nullptr && other.my_map_record_ptr == nullptr), + "invalid state of moving object?"); + + if (my_lru_cache_ptr) + my_lru_cache_ptr->signal_end_of_usage(*my_map_record_ptr); + + my_lru_cache_ptr = other.my_lru_cache_ptr; + my_map_record_ptr = other.my_map_record_ptr; + other.my_lru_cache_ptr = nullptr; + other.my_map_record_ptr = nullptr; + + return *this; + } + + ~handle_object() { + if (my_lru_cache_ptr) + my_lru_cache_ptr->signal_end_of_usage(*my_map_record_ptr); + } + + operator bool() const { + return (my_lru_cache_ptr && my_map_record_ptr); + } + + value_type& value() { + __TBB_ASSERT(my_lru_cache_ptr, "get value from already moved object?"); + __TBB_ASSERT(my_map_record_ptr, "get value from an invalid or already moved object?"); + + return my_map_record_ptr->second.my_value; + } +}; + +//----------------------------------------------------------------------------- +// Aggregator operation for aggregator type in concurrent LRU cache +//----------------------------------------------------------------------------- + +template +struct concurrent_lru_cache::aggregator_operation + : aggregated_operation { +// incapsulated helper classes +public: + enum class op_type { retrieve, signal_end_of_usage }; + +// fields +private: + op_type my_op; + +// interface +public: + aggregator_operation(op_type op) : my_op(op) {} + + // TODO: aggregator_operation can be implemented + // - as a statically typed variant type or CRTP? (static, dependent on the use case) + // - or use pointer to function and apply_visitor (dynamic) + // - or use virtual functions (dynamic) + void cast_and_handle(lru_cache_type& lru_cache_ref) { + if (my_op == op_type::retrieve) + static_cast(this)->handle(lru_cache_ref); + else + static_cast(this)->handle(lru_cache_ref); + } +}; + +template +struct concurrent_lru_cache::retrieve_aggregator_operation + : aggregator_operation, private no_assign { +public: + key_type my_key; + storage_map_pointer_type my_map_record_ptr; + bool my_is_new_value_needed; + +public: + retrieve_aggregator_operation(key_type key) + : aggregator_operation(aggregator_operation::op_type::retrieve), + my_key(key), my_map_record_ptr(nullptr), my_is_new_value_needed(false) {} + + void handle(lru_cache_type& lru_cache_ref) { + my_map_record_ptr = &lru_cache_ref.retrieve_serial(my_key, my_is_new_value_needed); + } + + storage_map_reference_type result() { + __TBB_ASSERT(my_map_record_ptr, "Attempt to call result() before calling handle()"); + return *my_map_record_ptr; + } + + bool is_new_value_needed() { return my_is_new_value_needed; } +}; + +template +struct concurrent_lru_cache::signal_end_of_usage_aggregator_operation + : aggregator_operation, private no_assign { + +private: + storage_map_reference_type my_map_record_ref; + +public: + signal_end_of_usage_aggregator_operation(storage_map_reference_type map_record_ref) + : aggregator_operation(aggregator_operation::op_type::signal_end_of_usage), + my_map_record_ref(map_record_ref) {} + + void handle(lru_cache_type& lru_cache_ref) { + lru_cache_ref.signal_end_of_usage_serial(my_map_record_ref); + } +}; + +// TODO: if we have guarantees that KeyToValFunctorT always have +// ValT as a return type and KeyT as an argument type +// we can deduce template parameters of concurrent_lru_cache +// by pattern matching on KeyToValFunctorT + +} // namespace d1 +} // namespace detail + +inline namespace v1 { + +using detail::d1::concurrent_lru_cache; + +} // inline namespace v1 +} // namespace tbb + +#endif // __TBB_concurrent_lru_cache_H diff --git a/src/3rd party/oneapi/tbb/concurrent_map.h b/src/3rd party/oneapi/tbb/concurrent_map.h new file mode 100644 index 0000000000..0dd196db14 --- /dev/null +++ b/src/3rd party/oneapi/tbb/concurrent_map.h @@ -0,0 +1,351 @@ +/* + Copyright (c) 2019-2025 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_map_H +#define __TBB_concurrent_map_H + +#include "detail/_namespace_injection.h" +#include "detail/_concurrent_skip_list.h" +#include "tbb_allocator.h" +#include +#include +#include + +namespace tbb { +namespace detail { +namespace d3 { + +template +struct map_traits { + static constexpr std::size_t max_level = RandomGenerator::max_level; + using random_level_generator_type = RandomGenerator; + using key_type = Key; + using mapped_type = Value; + using compare_type = KeyCompare; + using value_type = std::pair; + using reference = value_type&; + using const_reference = const value_type&; + using allocator_type = Allocator; + + static constexpr bool allow_multimapping = AllowMultimapping; + + class value_compare { + public: + bool operator()(const value_type& lhs, const value_type& rhs) const { + return comp(lhs.first, rhs.first); + } + + protected: + value_compare(compare_type c) : comp(c) {} + + friend struct map_traits; + + compare_type comp; + }; + + static value_compare value_comp(compare_type comp) { return value_compare(comp); } + + static const key_type& get_key(const_reference val) { + return val.first; + } +}; // struct map_traits + +template +class concurrent_multimap; + +template , typename Allocator = tbb::tbb_allocator>> +class concurrent_map : public concurrent_skip_list, Allocator, false>> { + using base_type = concurrent_skip_list, Allocator, false>>; +public: + using key_type = Key; + using mapped_type = Value; + using value_type = typename base_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Compare; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::const_pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + + using node_type = typename base_type::node_type; + + // Include constructors of base type + using base_type::base_type; + + // Required for implicit deduction guides + concurrent_map() = default; + concurrent_map( const concurrent_map& ) = default; + concurrent_map( const concurrent_map& other, const allocator_type& alloc ) : base_type(other, alloc) {} + concurrent_map( concurrent_map&& ) = default; + concurrent_map( concurrent_map&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {} + // Required to respect the rule of 5 + concurrent_map& operator=( const concurrent_map& ) = default; + concurrent_map& operator=( concurrent_map&& ) = default; + + concurrent_map& operator=( std::initializer_list il ) { + base_type::operator= (il); + return *this; + } + + // Observers + mapped_type& at(const key_type& key) { + iterator it = this->find(key); + + if (it == this->end()) { + throw_exception(exception_id::invalid_key); + } + return it->second; + } + + const mapped_type& at(const key_type& key) const { + return const_cast(this)->at(key); + } + + mapped_type& operator[](const key_type& key) { + iterator it = this->find(key); + + if (it == this->end()) { + it = this->emplace(std::piecewise_construct, std::forward_as_tuple(key), std::tuple<>()).first; + } + return it->second; + } + + mapped_type& operator[](key_type&& key) { + iterator it = this->find(key); + + if (it == this->end()) { + it = this->emplace(std::piecewise_construct, std::forward_as_tuple(std::move(key)), std::tuple<>()).first; + } + return it->second; + } + + using base_type::insert; + + template + typename std::enable_if::value, + std::pair>::type insert( P&& value ) + { + return this->emplace(std::forward

(value)); + } + + template + typename std::enable_if::value, + iterator>::type insert( const_iterator hint, P&& value ) + { + return this->emplace_hint(hint, std::forward

(value)); + } + + template + void merge(concurrent_map& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_map&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_multimap& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multimap&& source) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_map + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template >, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_map( It, It, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_map, iterator_mapped_t, Comp, Alloc>; + +template >, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_map( std::initializer_list>, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_map, T, Comp, Alloc>; + +template >, + typename = std::enable_if_t>> +concurrent_map( It, It, Alloc ) +-> concurrent_map, iterator_mapped_t, + std::less>, Alloc>; + +template >> +concurrent_map( std::initializer_list>, Alloc ) +-> concurrent_map, T, std::less>, Alloc>; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_map& lhs, + concurrent_map& rhs ) +{ + lhs.swap(rhs); +} + +template , typename Allocator = tbb::tbb_allocator>> +class concurrent_multimap : public concurrent_skip_list, Allocator, true>> { + using base_type = concurrent_skip_list, Allocator, true>>; +public: + using key_type = Key; + using mapped_type = Value; + using value_type = typename base_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Compare; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::const_pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + + using node_type = typename base_type::node_type; + + // Include constructors of base_type + using base_type::base_type; + using base_type::insert; + + // Required for implicit deduction guides + concurrent_multimap() = default; + concurrent_multimap( const concurrent_multimap& ) = default; + concurrent_multimap( const concurrent_multimap& other, const allocator_type& alloc ) : base_type(other, alloc) {} + concurrent_multimap( concurrent_multimap&& ) = default; + concurrent_multimap( concurrent_multimap&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {} + // Required to respect the rule of 5 + concurrent_multimap& operator=( const concurrent_multimap& ) = default; + concurrent_multimap& operator=( concurrent_multimap&& ) = default; + + concurrent_multimap& operator=( std::initializer_list il ) { + base_type::operator= (il); + return *this; + } + + template + typename std::enable_if::value, + std::pair>::type insert( P&& value ) + { + return this->emplace(std::forward

(value)); + } + + template + typename std::enable_if::value, + iterator>::type insert( const_iterator hint, P&& value ) + { + return this->emplace_hint(hint, std::forward

(value)); + } + + template + void merge(concurrent_multimap& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multimap&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_map& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_map&& source) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_multimap + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template >, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_multimap( It, It, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_multimap, iterator_mapped_t, Comp, Alloc>; + +template >, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_multimap( std::initializer_list>, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_multimap, T, Comp, Alloc>; + +template >, + typename = std::enable_if_t>> +concurrent_multimap( It, It, Alloc ) +-> concurrent_multimap, iterator_mapped_t, + std::less>, Alloc>; + +template >> +concurrent_multimap( std::initializer_list>, Alloc ) +-> concurrent_multimap, T, std::less>, Alloc>; + + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_multimap& lhs, + concurrent_multimap& rhs ) +{ + lhs.swap(rhs); +} + +} // namespace d3 +} // namespace detail + +inline namespace v1 { + +using detail::d3::concurrent_map; +using detail::d3::concurrent_multimap; +using detail::split; + +} // inline namespace v1 +} // namespace tbb + +#endif // __TBB_concurrent_map_H diff --git a/src/3rd party/oneapi/tbb/concurrent_priority_queue.h b/src/3rd party/oneapi/tbb/concurrent_priority_queue.h new file mode 100644 index 0000000000..6d7eb3eb9f --- /dev/null +++ b/src/3rd party/oneapi/tbb/concurrent_priority_queue.h @@ -0,0 +1,490 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_priority_queue_H +#define __TBB_concurrent_priority_queue_H + +#include "detail/_namespace_injection.h" +#include "detail/_aggregator.h" +#include "detail/_template_helpers.h" +#include "detail/_allocator_traits.h" +#include "detail/_range_common.h" +#include "detail/_exception.h" +#include "detail/_utils.h" +#include "detail/_containers_helpers.h" +#include "cache_aligned_allocator.h" +#include +#include +#include +#include +#include +#include + +namespace tbb { +namespace detail { +namespace d1 { + +template , typename Allocator = cache_aligned_allocator> +class concurrent_priority_queue { +public: + using value_type = T; + using reference = T&; + using const_reference = const T&; + + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + + using allocator_type = Allocator; + + concurrent_priority_queue() : concurrent_priority_queue(allocator_type{}) {} + + explicit concurrent_priority_queue( const allocator_type& alloc ) + : mark(0), my_size(0), my_compare(), data(alloc) + { + my_aggregator.initialize_handler(functor{this}); + } + + explicit concurrent_priority_queue( const Compare& compare, const allocator_type& alloc = allocator_type() ) + : mark(0), my_size(0), my_compare(compare), data(alloc) + { + my_aggregator.initialize_handler(functor{this}); + } + + explicit concurrent_priority_queue( size_type init_capacity, const allocator_type& alloc = allocator_type() ) + : mark(0), my_size(0), my_compare(), data(alloc) + { + data.reserve(init_capacity); + my_aggregator.initialize_handler(functor{this}); + } + + explicit concurrent_priority_queue( size_type init_capacity, const Compare& compare, const allocator_type& alloc = allocator_type() ) + : mark(0), my_size(0), my_compare(compare), data(alloc) + { + data.reserve(init_capacity); + my_aggregator.initialize_handler(functor{this}); + } + + template + concurrent_priority_queue( InputIterator begin, InputIterator end, const Compare& compare, const allocator_type& alloc = allocator_type() ) + : mark(0), my_compare(compare), data(begin, end, alloc) + { + my_aggregator.initialize_handler(functor{this}); + heapify(); + my_size.store(data.size(), std::memory_order_relaxed); + } + + template + concurrent_priority_queue( InputIterator begin, InputIterator end, const allocator_type& alloc = allocator_type() ) + : concurrent_priority_queue(begin, end, Compare(), alloc) {} + + concurrent_priority_queue( std::initializer_list init, const Compare& compare, const allocator_type& alloc = allocator_type() ) + : concurrent_priority_queue(init.begin(), init.end(), compare, alloc) {} + + concurrent_priority_queue( std::initializer_list init, const allocator_type& alloc = allocator_type() ) + : concurrent_priority_queue(init, Compare(), alloc) {} + + concurrent_priority_queue( const concurrent_priority_queue& other ) + : mark(other.mark), my_size(other.my_size.load(std::memory_order_relaxed)), my_compare(other.my_compare), + data(other.data) + { + my_aggregator.initialize_handler(functor{this}); + } + + concurrent_priority_queue( const concurrent_priority_queue& other, const allocator_type& alloc ) + : mark(other.mark), my_size(other.my_size.load(std::memory_order_relaxed)), my_compare(other.my_compare), + data(other.data, alloc) + { + my_aggregator.initialize_handler(functor{this}); + } + + concurrent_priority_queue( concurrent_priority_queue&& other ) + : mark(other.mark), my_size(other.my_size.load(std::memory_order_relaxed)), my_compare(other.my_compare), + data(std::move(other.data)) + { + my_aggregator.initialize_handler(functor{this}); + } + + concurrent_priority_queue( concurrent_priority_queue&& other, const allocator_type& alloc ) + : mark(other.mark), my_size(other.my_size.load(std::memory_order_relaxed)), my_compare(other.my_compare), + data(std::move(other.data), alloc) + { + my_aggregator.initialize_handler(functor{this}); + } + + concurrent_priority_queue& operator=( const concurrent_priority_queue& other ) { + if (this != &other) { + data = other.data; + mark = other.mark; + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + } + return *this; + } + + concurrent_priority_queue& operator=( concurrent_priority_queue&& other ) { + if (this != &other) { + // TODO: check if exceptions from std::vector::operator=(vector&&) should be handled separately + data = std::move(other.data); + mark = other.mark; + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + } + return *this; + } + + concurrent_priority_queue& operator=( std::initializer_list init ) { + assign(init.begin(), init.end()); + return *this; + } + + template + void assign( InputIterator begin, InputIterator end ) { + data.assign(begin, end); + mark = 0; + my_size.store(data.size(), std::memory_order_relaxed); + heapify(); + } + + void assign( std::initializer_list init ) { + assign(init.begin(), init.end()); + } + + /* Returned value may not reflect results of pending operations. + This operation reads shared data and will trigger a race condition. */ + __TBB_nodiscard bool empty() const { return size() == 0; } + + // Returns the current number of elements contained in the queue + /* Returned value may not reflect results of pending operations. + This operation reads shared data and will trigger a race condition. */ + size_type size() const { return my_size.load(std::memory_order_relaxed); } + + /* This operation can be safely used concurrently with other push, try_pop or emplace operations. */ + void push( const value_type& value ) { + cpq_operation op_data(value, PUSH_OP); + my_aggregator.execute(&op_data); + if (op_data.status == FAILED) + throw_exception(exception_id::bad_alloc); + } + + /* This operation can be safely used concurrently with other push, try_pop or emplace operations. */ + void push( value_type&& value ) { + cpq_operation op_data(value, PUSH_RVALUE_OP); + my_aggregator.execute(&op_data); + if (op_data.status == FAILED) + throw_exception(exception_id::bad_alloc); + } + + /* This operation can be safely used concurrently with other push, try_pop or emplace operations. */ + template + void emplace( Args&&... args ) { + // TODO: support uses allocator construction in this place + push(value_type(std::forward(args)...)); + } + + // Gets a reference to and removes highest priority element + /* If a highest priority element was found, sets elem and returns true, + otherwise returns false. + This operation can be safely used concurrently with other push, try_pop or emplace operations. */ + bool try_pop( value_type& value ) { + cpq_operation op_data(value, POP_OP); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + + // This operation affects the whole container => it is not thread-safe + void clear() { + data.clear(); + mark = 0; + my_size.store(0, std::memory_order_relaxed); + } + + // This operation affects the whole container => it is not thread-safe + void swap( concurrent_priority_queue& other ) { + if (this != &other) { + using std::swap; + swap(data, other.data); + swap(mark, other.mark); + + size_type sz = my_size.load(std::memory_order_relaxed); + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.my_size.store(sz, std::memory_order_relaxed); + } + } + + allocator_type get_allocator() const { return data.get_allocator(); } +private: + enum operation_type {INVALID_OP, PUSH_OP, POP_OP, PUSH_RVALUE_OP}; + enum operation_status {WAIT = 0, SUCCEEDED, FAILED}; + + class cpq_operation : public aggregated_operation { + public: + operation_type type; + union { + value_type* elem; + size_type sz; + }; + cpq_operation( const value_type& value, operation_type t ) + : type(t), elem(const_cast(&value)) {} + }; // class cpq_operation + + class functor { + concurrent_priority_queue* my_cpq; + public: + functor() : my_cpq(nullptr) {} + functor( concurrent_priority_queue* cpq ) : my_cpq(cpq) {} + + void operator()(cpq_operation* op_list) { + __TBB_ASSERT(my_cpq != nullptr, "Invalid functor"); + my_cpq->handle_operations(op_list); + } + }; // class functor + + void handle_operations( cpq_operation* op_list ) { + call_itt_notify(acquired, this); + cpq_operation* tmp, *pop_list = nullptr; + __TBB_ASSERT(mark == data.size(), nullptr); + + // First pass processes all constant (amortized; reallocation may happen) time pushes and pops. + while(op_list) { + // ITT note: &(op_list->status) tag is used to cover accesses to op_list + // node. This thread is going to handle the operation, and so will acquire it + // and perform the associated operation w/o triggering a race condition; the + // thread that created the operation is waiting on the status field, so when + // this thread is done with the operation, it will perform a + // store_with_release to give control back to the waiting thread in + // aggregator::insert_operation. + // TODO: enable + call_itt_notify(acquired, &(op_list->status)); + __TBB_ASSERT(op_list->type != INVALID_OP, nullptr); + + tmp = op_list; + op_list = op_list->next.load(std::memory_order_relaxed); + if (tmp->type == POP_OP) { + if (mark < data.size() && + my_compare(data[0], data.back())) + { + // there are newly pushed elems and the last one is higher than top + *(tmp->elem) = std::move(data.back()); + my_size.store(my_size.load(std::memory_order_relaxed) - 1, std::memory_order_relaxed); + tmp->status.store(uintptr_t(SUCCEEDED), std::memory_order_release); + + data.pop_back(); + __TBB_ASSERT(mark <= data.size(), nullptr); + } else { // no convenient item to pop; postpone + tmp->next.store(pop_list, std::memory_order_relaxed); + pop_list = tmp; + } + } else { // PUSH_OP or PUSH_RVALUE_OP + __TBB_ASSERT(tmp->type == PUSH_OP || tmp->type == PUSH_RVALUE_OP, "Unknown operation"); +#if TBB_USE_EXCEPTIONS + try +#endif + { + if (tmp->type == PUSH_OP) { + push_back_helper(*(tmp->elem)); + } else { + data.push_back(std::move(*(tmp->elem))); + } + my_size.store(my_size.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); + tmp->status.store(uintptr_t(SUCCEEDED), std::memory_order_release); + } +#if TBB_USE_EXCEPTIONS + catch(...) { + tmp->status.store(uintptr_t(FAILED), std::memory_order_release); + } +#endif + } + } + + // Second pass processes pop operations + while(pop_list) { + tmp = pop_list; + pop_list = pop_list->next.load(std::memory_order_relaxed); + __TBB_ASSERT(tmp->type == POP_OP, nullptr); + if (data.empty()) { + tmp->status.store(uintptr_t(FAILED), std::memory_order_release); + } else { + __TBB_ASSERT(mark <= data.size(), nullptr); + if (mark < data.size() && + my_compare(data[0], data.back())) + { + // there are newly pushed elems and the last one is higher than top + *(tmp->elem) = std::move(data.back()); + my_size.store(my_size.load(std::memory_order_relaxed) - 1, std::memory_order_relaxed); + tmp->status.store(uintptr_t(SUCCEEDED), std::memory_order_release); + data.pop_back(); + } else { // extract top and push last element down heap + *(tmp->elem) = std::move(data[0]); + my_size.store(my_size.load(std::memory_order_relaxed) - 1, std::memory_order_relaxed); + tmp->status.store(uintptr_t(SUCCEEDED), std::memory_order_release); + reheap(); + } + } + } + + // heapify any leftover pushed elements before doing the next + // batch of operations + if (mark < data.size()) heapify(); + __TBB_ASSERT(mark == data.size(), nullptr); + call_itt_notify(releasing, this); + } + + // Merge unsorted elements into heap + void heapify() { + if (!mark && data.size() > 0) mark = 1; + for (; mark < data.size(); ++mark) { + // for each unheapified element under size + size_type cur_pos = mark; + value_type to_place = std::move(data[mark]); + do { // push to_place up the heap + size_type parent = (cur_pos - 1) >> 1; + if (!my_compare(data[parent], to_place)) + break; + data[cur_pos] = std::move(data[parent]); + cur_pos = parent; + } while(cur_pos); + data[cur_pos] = std::move(to_place); + } + } + + // Re-heapify after an extraction + // Re-heapify by pushing last element down the heap from the root. + void reheap() { + size_type cur_pos = 0, child = 1; + + while(child < mark) { + size_type target = child; + if (child + 1 < mark && my_compare(data[child], data[child + 1])) + ++target; + // target now has the higher priority child + if (my_compare(data[target], data.back())) + break; + data[cur_pos] = std::move(data[target]); + cur_pos = target; + child = (cur_pos << 1) + 1; + } + if (cur_pos != data.size() - 1) + data[cur_pos] = std::move(data.back()); + data.pop_back(); + if (mark > data.size()) mark = data.size(); + } + + void push_back_helper( const T& value ) { + push_back_helper_impl(value, std::is_copy_constructible{}); + } + + void push_back_helper_impl( const T& value, /*is_copy_constructible = */std::true_type ) { + data.push_back(value); + } + + void push_back_helper_impl( const T&, /*is_copy_constructible = */std::false_type ) { + __TBB_ASSERT(false, "error: calling tbb::concurrent_priority_queue.push(const value_type&) for move-only type"); + } + + using aggregator_type = aggregator; + + aggregator_type my_aggregator; + // Padding added to avoid false sharing + char padding1[max_nfs_size - sizeof(aggregator_type)]; + // The point at which unsorted elements begin + size_type mark; + std::atomic my_size; + Compare my_compare; + + // Padding added to avoid false sharing + char padding2[max_nfs_size - (2*sizeof(size_type)) - sizeof(Compare)]; + //! Storage for the heap of elements in queue, plus unheapified elements + /** data has the following structure: + + binary unheapified + heap elements + ____|_______|____ + | | | + v v v + [_|...|_|_|...|_| |...| ] + 0 ^ ^ ^ + | | |__capacity + | |__my_size + |__mark + + Thus, data stores the binary heap starting at position 0 through + mark-1 (it may be empty). Then there are 0 or more elements + that have not yet been inserted into the heap, in positions + mark through my_size-1. */ + + using vector_type = std::vector; + vector_type data; + + friend bool operator==( const concurrent_priority_queue& lhs, + const concurrent_priority_queue& rhs ) + { + return lhs.data == rhs.data; + } + +#if !__TBB_CPP20_COMPARISONS_PRESENT + friend bool operator!=( const concurrent_priority_queue& lhs, + const concurrent_priority_queue& rhs ) + { + return !(lhs == rhs); + } +#endif +}; // class concurrent_priority_queue + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +template >, + typename Alloc = tbb::cache_aligned_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_priority_queue( It, It, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_priority_queue, Comp, Alloc>; + +template >, + typename = std::enable_if_t>> +concurrent_priority_queue( It, It, Alloc ) +-> concurrent_priority_queue, std::less>, Alloc>; + +template , + typename Alloc = tbb::cache_aligned_allocator, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_priority_queue( std::initializer_list, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_priority_queue; + +template >> +concurrent_priority_queue( std::initializer_list, Alloc ) +-> concurrent_priority_queue, Alloc>; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_priority_queue& lhs, + concurrent_priority_queue& rhs ) +{ + lhs.swap(rhs); +} + +} // namespace d1 +} // namespace detail +inline namespace v1 { +using detail::d1::concurrent_priority_queue; + +} // inline namespace v1 +} // namespace tbb + +#endif // __TBB_concurrent_priority_queue_H diff --git a/src/3rd party/oneapi/tbb/concurrent_queue.h b/src/3rd party/oneapi/tbb/concurrent_queue.h new file mode 100644 index 0000000000..ea61b68b60 --- /dev/null +++ b/src/3rd party/oneapi/tbb/concurrent_queue.h @@ -0,0 +1,705 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_queue_H +#define __TBB_concurrent_queue_H + +#include "detail/_namespace_injection.h" +#include "detail/_concurrent_queue_base.h" +#include "detail/_allocator_traits.h" +#include "detail/_exception.h" +#include "detail/_containers_helpers.h" +#include "cache_aligned_allocator.h" + +namespace tbb { +namespace detail { +namespace d2 { + +template +std::pair internal_try_pop_impl(void* dst, QueueRep& queue, Allocator& alloc ) { + ticket_type ticket{}; + do { + // Basically, we need to read `head_counter` before `tail_counter`. To achieve it we build happens-before on `head_counter` + ticket = queue.head_counter.load(std::memory_order_acquire); + do { + if (static_cast(queue.tail_counter.load(std::memory_order_relaxed) - ticket) <= 0) { // queue is empty + // Queue is empty + return { false, ticket }; + } + // Queue had item with ticket k when we looked. Attempt to get that item. + // Another thread snatched the item, retry. + } while (!queue.head_counter.compare_exchange_strong(ticket, ticket + 1)); + } while (!queue.choose(ticket).pop(dst, ticket, queue, alloc)); + return { true, ticket }; +} + +// A high-performance thread-safe non-blocking concurrent queue. +// Multiple threads may each push and pop concurrently. +template > +class concurrent_queue { + using allocator_traits_type = tbb::detail::allocator_traits; + using queue_representation_type = concurrent_queue_rep; + using queue_allocator_type = typename allocator_traits_type::template rebind_alloc; + using queue_allocator_traits = tbb::detail::allocator_traits; +public: + using size_type = std::size_t; + using value_type = T; + using reference = T&; + using const_reference = const T&; + using difference_type = std::ptrdiff_t; + + using allocator_type = Allocator; + using pointer = typename allocator_traits_type::pointer; + using const_pointer = typename allocator_traits_type::const_pointer; + + using iterator = concurrent_queue_iterator; + using const_iterator = concurrent_queue_iterator; + + concurrent_queue() : concurrent_queue(allocator_type()) {} + + explicit concurrent_queue(const allocator_type& a) : + my_allocator(a), my_queue_representation(nullptr) + { + my_queue_representation = static_cast(r1::cache_aligned_allocate(sizeof(queue_representation_type))); + queue_allocator_traits::construct(my_allocator, my_queue_representation); + + __TBB_ASSERT(is_aligned(my_queue_representation, max_nfs_size), "alignment error" ); + __TBB_ASSERT(is_aligned(&my_queue_representation->head_counter, max_nfs_size), "alignment error" ); + __TBB_ASSERT(is_aligned(&my_queue_representation->tail_counter, max_nfs_size), "alignment error" ); + __TBB_ASSERT(is_aligned(&my_queue_representation->array, max_nfs_size), "alignment error" ); + } + + template + concurrent_queue(InputIterator begin, InputIterator end, const allocator_type& a = allocator_type()) : + concurrent_queue(a) + { + for (; begin != end; ++begin) + push(*begin); + } + + concurrent_queue( std::initializer_list init, const allocator_type& alloc = allocator_type() ) : + concurrent_queue(init.begin(), init.end(), alloc) + {} + + concurrent_queue(const concurrent_queue& src, const allocator_type& a) : + concurrent_queue(a) + { + my_queue_representation->assign(*src.my_queue_representation, my_allocator, copy_construct_item); + } + + concurrent_queue(const concurrent_queue& src) : + concurrent_queue(queue_allocator_traits::select_on_container_copy_construction(src.get_allocator())) + { + my_queue_representation->assign(*src.my_queue_representation, my_allocator, copy_construct_item); + } + + // Move constructors + concurrent_queue(concurrent_queue&& src) : + concurrent_queue(std::move(src.my_allocator)) + { + internal_swap(src); + } + + concurrent_queue(concurrent_queue&& src, const allocator_type& a) : + concurrent_queue(a) + { + // checking that memory allocated by one instance of allocator can be deallocated + // with another + if (my_allocator == src.my_allocator) { + internal_swap(src); + } else { + // allocators are different => performing per-element move + my_queue_representation->assign(*src.my_queue_representation, my_allocator, move_construct_item); + src.clear(); + } + } + + // Destroy queue + ~concurrent_queue() { + clear(); + my_queue_representation->clear(my_allocator); + queue_allocator_traits::destroy(my_allocator, my_queue_representation); + r1::cache_aligned_deallocate(my_queue_representation); + } + + concurrent_queue& operator=( const concurrent_queue& other ) { + //TODO: implement support for std::allocator_traits::propagate_on_container_copy_assignment + if (my_queue_representation != other.my_queue_representation) { + clear(); + my_allocator = other.my_allocator; + my_queue_representation->assign(*other.my_queue_representation, my_allocator, copy_construct_item); + } + return *this; + } + + concurrent_queue& operator=( concurrent_queue&& other ) { + //TODO: implement support for std::allocator_traits::propagate_on_container_move_assignment + if (my_queue_representation != other.my_queue_representation) { + clear(); + if (my_allocator == other.my_allocator) { + internal_swap(other); + } else { + my_queue_representation->assign(*other.my_queue_representation, other.my_allocator, move_construct_item); + other.clear(); + my_allocator = std::move(other.my_allocator); + } + } + return *this; + } + + concurrent_queue& operator=( std::initializer_list init ) { + assign(init); + return *this; + } + + template + void assign( InputIterator first, InputIterator last ) { + concurrent_queue src(first, last); + clear(); + my_queue_representation->assign(*src.my_queue_representation, my_allocator, move_construct_item); + } + + void assign( std::initializer_list init ) { + assign(init.begin(), init.end()); + } + + void swap ( concurrent_queue& other ) { + //TODO: implement support for std::allocator_traits::propagate_on_container_swap + __TBB_ASSERT(my_allocator == other.my_allocator, "unequal allocators"); + internal_swap(other); + } + + // Enqueue an item at tail of queue. + void push(const T& value) { + internal_push(value); + } + + void push(T&& value) { + internal_push(std::move(value)); + } + + template + void emplace( Args&&... args ) { + internal_push(std::forward(args)...); + } + + // Attempt to dequeue an item from head of queue. + /** Does not wait for item to become available. + Returns true if successful; false otherwise. */ + bool try_pop( T& result ) { + return internal_try_pop(&result); + } + + // Return the number of items in the queue; thread unsafe + size_type unsafe_size() const { + std::ptrdiff_t size = my_queue_representation->size(); + return size < 0 ? 0 : size_type(size); + } + + // Equivalent to size()==0. + __TBB_nodiscard bool empty() const { + return my_queue_representation->empty(); + } + + // Clear the queue. not thread-safe. + void clear() { + my_queue_representation->clear(my_allocator); + } + + // Return allocator object + allocator_type get_allocator() const { return my_allocator; } + + //------------------------------------------------------------------------ + // The iterators are intended only for debugging. They are slow and not thread safe. + //------------------------------------------------------------------------ + + iterator unsafe_begin() { return concurrent_queue_iterator_provider::get(*this); } + iterator unsafe_end() { return iterator(); } + const_iterator unsafe_begin() const { return concurrent_queue_iterator_provider::get(*this); } + const_iterator unsafe_end() const { return const_iterator(); } + const_iterator unsafe_cbegin() const { return concurrent_queue_iterator_provider::get(*this); } + const_iterator unsafe_cend() const { return const_iterator(); } + +private: + void internal_swap(concurrent_queue& src) { + using std::swap; + swap(my_queue_representation, src.my_queue_representation); + } + + template + void internal_push( Args&&... args ) { + ticket_type k = my_queue_representation->tail_counter++; + my_queue_representation->choose(k).push(k, *my_queue_representation, my_allocator, std::forward(args)...); + } + + bool internal_try_pop( void* dst ) { + return internal_try_pop_impl(dst, *my_queue_representation, my_allocator).first; + } + + template + friend class concurrent_queue_iterator; + + static void copy_construct_item(T* location, const void* src) { + // TODO: use allocator_traits for copy construction + new (location) value_type(*static_cast(src)); + // queue_allocator_traits::construct(my_allocator, location, *static_cast(src)); + } + + static void move_construct_item(T* location, const void* src) { + // TODO: use allocator_traits for move construction + new (location) value_type(std::move(*static_cast(const_cast(src)))); + } + + queue_allocator_type my_allocator; + queue_representation_type* my_queue_representation; + + friend void swap( concurrent_queue& lhs, concurrent_queue& rhs ) { + lhs.swap(rhs); + } + + friend bool operator==( const concurrent_queue& lhs, const concurrent_queue& rhs ) { + return lhs.unsafe_size() == rhs.unsafe_size() && std::equal(lhs.unsafe_begin(), lhs.unsafe_end(), rhs.unsafe_begin()); + } + +#if !__TBB_CPP20_COMPARISONS_PRESENT + friend bool operator!=( const concurrent_queue& lhs, const concurrent_queue& rhs ) { + return !(lhs == rhs); + } +#endif // __TBB_CPP20_COMPARISONS_PRESENT +}; // class concurrent_queue + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +// Deduction guide for the constructor from two iterators +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_queue( It, It, Alloc = Alloc() ) +-> concurrent_queue, Alloc>; + +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + +class concurrent_monitor; + +// The concurrent monitor tags for concurrent_bounded_queue. +__TBB_GLOBAL_VAR constexpr std::size_t cbq_slots_avail_tag = 0; +__TBB_GLOBAL_VAR constexpr std::size_t cbq_items_avail_tag = 1; +} // namespace d2 + + +namespace r1 { + class concurrent_monitor; + + TBB_EXPORT std::uint8_t* __TBB_EXPORTED_FUNC allocate_bounded_queue_rep( std::size_t queue_rep_size ); + TBB_EXPORT void __TBB_EXPORTED_FUNC deallocate_bounded_queue_rep( std::uint8_t* mem, std::size_t queue_rep_size ); + TBB_EXPORT void __TBB_EXPORTED_FUNC abort_bounded_queue_monitors( concurrent_monitor* monitors ); + TBB_EXPORT void __TBB_EXPORTED_FUNC notify_bounded_queue_monitor( concurrent_monitor* monitors, std::size_t monitor_tag + , std::size_t ticket ); + TBB_EXPORT void __TBB_EXPORTED_FUNC wait_bounded_queue_monitor( concurrent_monitor* monitors, std::size_t monitor_tag, + std::ptrdiff_t target, d1::delegate_base& predicate ); +} // namespace r1 + + +namespace d2 { +// A high-performance thread-safe blocking concurrent bounded queue. +// Supports boundedness and blocking semantics. +// Multiple threads may each push and pop concurrently. +template > +class concurrent_bounded_queue { + using allocator_traits_type = tbb::detail::allocator_traits; + using queue_representation_type = concurrent_queue_rep; + using queue_allocator_type = typename allocator_traits_type::template rebind_alloc; + using queue_allocator_traits = tbb::detail::allocator_traits; + + template + void internal_wait(r1::concurrent_monitor* monitors, std::size_t monitor_tag, std::ptrdiff_t target, FuncType pred) { + d1::delegated_function func(pred); + r1::wait_bounded_queue_monitor(monitors, monitor_tag, target, func); + } +public: + using size_type = std::ptrdiff_t; + using value_type = T; + using reference = T&; + using const_reference = const T&; + using difference_type = std::ptrdiff_t; + + using allocator_type = Allocator; + using pointer = typename allocator_traits_type::pointer; + using const_pointer = typename allocator_traits_type::const_pointer; + + using iterator = concurrent_queue_iterator; + using const_iterator = concurrent_queue_iterator ; + + concurrent_bounded_queue() : concurrent_bounded_queue(allocator_type()) {} + + explicit concurrent_bounded_queue( const allocator_type& a ) : + my_allocator(a), my_capacity(0), my_abort_counter(0), my_queue_representation(nullptr) + { + my_queue_representation = reinterpret_cast( + r1::allocate_bounded_queue_rep(sizeof(queue_representation_type))); + my_monitors = reinterpret_cast(my_queue_representation + 1); + queue_allocator_traits::construct(my_allocator, my_queue_representation); + my_capacity = std::size_t(-1) / (queue_representation_type::item_size > 1 ? queue_representation_type::item_size : 2); + + __TBB_ASSERT(is_aligned(my_queue_representation, max_nfs_size), "alignment error" ); + __TBB_ASSERT(is_aligned(&my_queue_representation->head_counter, max_nfs_size), "alignment error" ); + __TBB_ASSERT(is_aligned(&my_queue_representation->tail_counter, max_nfs_size), "alignment error" ); + __TBB_ASSERT(is_aligned(&my_queue_representation->array, max_nfs_size), "alignment error" ); + } + + template + concurrent_bounded_queue( InputIterator begin, InputIterator end, const allocator_type& a = allocator_type() ) : + concurrent_bounded_queue(a) + { + for (; begin != end; ++begin) + push(*begin); + } + + concurrent_bounded_queue( std::initializer_list init, const allocator_type& alloc = allocator_type() ): + concurrent_bounded_queue(init.begin(), init.end(), alloc) + {} + + concurrent_bounded_queue( const concurrent_bounded_queue& src, const allocator_type& a ) : + concurrent_bounded_queue(a) + { + my_capacity = src.my_capacity; + my_queue_representation->assign(*src.my_queue_representation, my_allocator, copy_construct_item); + } + + concurrent_bounded_queue( const concurrent_bounded_queue& src ) : + concurrent_bounded_queue(queue_allocator_traits::select_on_container_copy_construction(src.get_allocator())) + { + my_capacity = src.my_capacity; + my_queue_representation->assign(*src.my_queue_representation, my_allocator, copy_construct_item); + } + + // Move constructors + concurrent_bounded_queue( concurrent_bounded_queue&& src ) : + concurrent_bounded_queue(std::move(src.my_allocator)) + { + internal_swap(src); + } + + concurrent_bounded_queue( concurrent_bounded_queue&& src, const allocator_type& a ) : + concurrent_bounded_queue(a) + { + // checking that memory allocated by one instance of allocator can be deallocated + // with another + if (my_allocator == src.my_allocator) { + internal_swap(src); + } else { + // allocators are different => performing per-element move + my_queue_representation->assign(*src.my_queue_representation, my_allocator, move_construct_item); + src.clear(); + } + } + + // Destroy queue + ~concurrent_bounded_queue() { + clear(); + my_queue_representation->clear(my_allocator); + queue_allocator_traits::destroy(my_allocator, my_queue_representation); + r1::deallocate_bounded_queue_rep(reinterpret_cast(my_queue_representation), + sizeof(queue_representation_type)); + } + + concurrent_bounded_queue& operator=( const concurrent_bounded_queue& other ) { + //TODO: implement support for std::allocator_traits::propagate_on_container_copy_assignment + if (my_queue_representation != other.my_queue_representation) { + clear(); + my_allocator = other.my_allocator; + my_capacity = other.my_capacity; + my_queue_representation->assign(*other.my_queue_representation, my_allocator, copy_construct_item); + } + return *this; + } + + concurrent_bounded_queue& operator=( concurrent_bounded_queue&& other ) { + //TODO: implement support for std::allocator_traits::propagate_on_container_move_assignment + if (my_queue_representation != other.my_queue_representation) { + clear(); + if (my_allocator == other.my_allocator) { + internal_swap(other); + } else { + my_queue_representation->assign(*other.my_queue_representation, other.my_allocator, move_construct_item); + other.clear(); + my_allocator = std::move(other.my_allocator); + my_capacity = other.my_capacity; + } + } + return *this; + } + + concurrent_bounded_queue& operator=( std::initializer_list init ) { + assign(init); + return *this; + } + + template + void assign( InputIterator first, InputIterator last ) { + concurrent_bounded_queue src(first, last); + clear(); + my_queue_representation->assign(*src.my_queue_representation, my_allocator, move_construct_item); + } + + void assign( std::initializer_list init ) { + assign(init.begin(), init.end()); + } + + void swap ( concurrent_bounded_queue& other ) { + //TODO: implement support for std::allocator_traits::propagate_on_container_swap + __TBB_ASSERT(my_allocator == other.my_allocator, "unequal allocators"); + internal_swap(other); + } + + // Enqueue an item at tail of queue. + void push( const T& value ) { + internal_push(value); + } + + void push( T&& value ) { + internal_push(std::move(value)); + } + + // Enqueue an item at tail of queue if queue is not already full. + // Does not wait for queue to become not full. + // Returns true if item is pushed; false if queue was already full. + bool try_push( const T& value ) { + return internal_push_if_not_full(value); + } + + bool try_push( T&& value ) { + return internal_push_if_not_full(std::move(value)); + } + + template + void emplace( Args&&... args ) { + internal_push(std::forward(args)...); + } + + template + bool try_emplace( Args&&... args ) { + return internal_push_if_not_full(std::forward(args)...); + } + + // Attempt to dequeue an item from head of queue. + void pop( T& result ) { + internal_pop(&result); + } + + /** Does not wait for item to become available. + Returns true if successful; false otherwise. */ + bool try_pop( T& result ) { + return internal_pop_if_present(&result); + } + + void abort() { + internal_abort(); + } + + // Return the number of items in the queue; thread unsafe + std::ptrdiff_t size() const { + return my_queue_representation->size(); + } + + void set_capacity( size_type new_capacity ) { + std::ptrdiff_t c = new_capacity < 0 ? infinite_capacity : new_capacity; + my_capacity = c; + } + + size_type capacity() const { + return my_capacity; + } + + // Equivalent to size()==0. + __TBB_nodiscard bool empty() const { + return my_queue_representation->empty(); + } + + // Clear the queue. not thread-safe. + void clear() { + my_queue_representation->clear(my_allocator); + } + + // Return allocator object + allocator_type get_allocator() const { return my_allocator; } + + //------------------------------------------------------------------------ + // The iterators are intended only for debugging. They are slow and not thread safe. + //------------------------------------------------------------------------ + + iterator unsafe_begin() { return concurrent_queue_iterator_provider::get(*this); } + iterator unsafe_end() { return iterator(); } + const_iterator unsafe_begin() const { return concurrent_queue_iterator_provider::get(*this); } + const_iterator unsafe_end() const { return const_iterator(); } + const_iterator unsafe_cbegin() const { return concurrent_queue_iterator_provider::get(*this); } + const_iterator unsafe_cend() const { return const_iterator(); } + +private: + void internal_swap( concurrent_bounded_queue& src ) { + using std::swap; + swap(my_queue_representation, src.my_queue_representation); + swap(my_capacity, src.my_capacity); + swap(my_monitors, src.my_monitors); + } + + static constexpr std::ptrdiff_t infinite_capacity = std::ptrdiff_t(~size_type(0) / 2); + + template + void internal_push( Args&&... args ) { + unsigned old_abort_counter = my_abort_counter.load(std::memory_order_relaxed); + ticket_type ticket = my_queue_representation->tail_counter++; + std::ptrdiff_t target = ticket - my_capacity; + + if (static_cast(my_queue_representation->head_counter.load(std::memory_order_relaxed)) <= target) { // queue is full + auto pred = [&] { + if (my_abort_counter.load(std::memory_order_relaxed) != old_abort_counter) { + throw_exception(exception_id::user_abort); + } + + return static_cast(my_queue_representation->head_counter.load(std::memory_order_relaxed)) <= target; + }; + + try_call( [&] { + internal_wait(my_monitors, cbq_slots_avail_tag, target, pred); + }).on_exception( [&] { + my_queue_representation->choose(ticket).abort_push(ticket, *my_queue_representation, my_allocator); + }); + + } + __TBB_ASSERT((static_cast(my_queue_representation->head_counter.load(std::memory_order_relaxed)) > target), nullptr); + my_queue_representation->choose(ticket).push(ticket, *my_queue_representation, my_allocator, std::forward(args)...); + r1::notify_bounded_queue_monitor(my_monitors, cbq_items_avail_tag, ticket); + } + + template + bool internal_push_if_not_full( Args&&... args ) { + ticket_type ticket = my_queue_representation->tail_counter.load(std::memory_order_relaxed); + do { + if (static_cast(ticket - my_queue_representation->head_counter.load(std::memory_order_relaxed)) >= my_capacity) { + // Queue is full + return false; + } + // Queue had empty slot with ticket k when we looked. Attempt to claim that slot. + // Another thread claimed the slot, so retry. + } while (!my_queue_representation->tail_counter.compare_exchange_strong(ticket, ticket + 1)); + + my_queue_representation->choose(ticket).push(ticket, *my_queue_representation, my_allocator, std::forward(args)...); + r1::notify_bounded_queue_monitor(my_monitors, cbq_items_avail_tag, ticket); + return true; + } + + void internal_pop( void* dst ) { + std::ptrdiff_t target; + // This loop is a single pop operation; abort_counter should not be re-read inside + unsigned old_abort_counter = my_abort_counter.load(std::memory_order_relaxed); + + do { + target = my_queue_representation->head_counter++; + if (static_cast(my_queue_representation->tail_counter.load(std::memory_order_relaxed)) <= target) { + auto pred = [&] { + if (my_abort_counter.load(std::memory_order_relaxed) != old_abort_counter) { + throw_exception(exception_id::user_abort); + } + + return static_cast(my_queue_representation->tail_counter.load(std::memory_order_relaxed)) <= target; + }; + + try_call( [&] { + internal_wait(my_monitors, cbq_items_avail_tag, target, pred); + }).on_exception( [&] { + my_queue_representation->head_counter--; + }); + } + __TBB_ASSERT(static_cast(my_queue_representation->tail_counter.load(std::memory_order_relaxed)) > target, nullptr); + } while (!my_queue_representation->choose(target).pop(dst, target, *my_queue_representation, my_allocator)); + + r1::notify_bounded_queue_monitor(my_monitors, cbq_slots_avail_tag, target); + } + + bool internal_pop_if_present( void* dst ) { + bool present{}; + ticket_type ticket{}; + std::tie(present, ticket) = internal_try_pop_impl(dst, *my_queue_representation, my_allocator); + + if (present) { + r1::notify_bounded_queue_monitor(my_monitors, cbq_slots_avail_tag, ticket); + } + return present; + } + + void internal_abort() { + ++my_abort_counter; + r1::abort_bounded_queue_monitors(my_monitors); + } + + static void copy_construct_item(T* location, const void* src) { + // TODO: use allocator_traits for copy construction + new (location) value_type(*static_cast(src)); + } + + static void move_construct_item(T* location, const void* src) { + // TODO: use allocator_traits for move construction + new (location) value_type(std::move(*static_cast(const_cast(src)))); + } + + template + friend class concurrent_queue_iterator; + + queue_allocator_type my_allocator; + std::ptrdiff_t my_capacity; + std::atomic my_abort_counter; + queue_representation_type* my_queue_representation; + + r1::concurrent_monitor* my_monitors; + + friend void swap( concurrent_bounded_queue& lhs, concurrent_bounded_queue& rhs ) { + lhs.swap(rhs); + } + + friend bool operator==( const concurrent_bounded_queue& lhs, const concurrent_bounded_queue& rhs ) { + return lhs.size() == rhs.size() && std::equal(lhs.unsafe_begin(), lhs.unsafe_end(), rhs.unsafe_begin()); + } + +#if !__TBB_CPP20_COMPARISONS_PRESENT + friend bool operator!=( const concurrent_bounded_queue& lhs, const concurrent_bounded_queue& rhs ) { + return !(lhs == rhs); + } +#endif // __TBB_CPP20_COMPARISONS_PRESENT +}; // class concurrent_bounded_queue + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +// Deduction guide for the constructor from two iterators +template >> +concurrent_bounded_queue( It, It, Alloc = Alloc() ) +-> concurrent_bounded_queue, Alloc>; + +#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ + +} //namespace d2 +} // namespace detail + +inline namespace v1 { + +using detail::d2::concurrent_queue; +using detail::d2::concurrent_bounded_queue; +using detail::r1::user_abort; +using detail::r1::bad_last_alloc; + +} // inline namespace v1 +} // namespace tbb + +#endif // __TBB_concurrent_queue_H diff --git a/src/3rd party/oneapi/tbb/concurrent_set.h b/src/3rd party/oneapi/tbb/concurrent_set.h new file mode 100644 index 0000000000..f9ad893d92 --- /dev/null +++ b/src/3rd party/oneapi/tbb/concurrent_set.h @@ -0,0 +1,268 @@ +/* + Copyright (c) 2019-2025 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_set_H +#define __TBB_concurrent_set_H + +#include "detail/_namespace_injection.h" +#include "detail/_concurrent_skip_list.h" +#include "tbb_allocator.h" +#include +#include + +namespace tbb { +namespace detail { +namespace d3 { + +template +struct set_traits { + static constexpr std::size_t max_level = RandomGenerator::max_level; + using random_level_generator_type = RandomGenerator; + using key_type = Key; + using value_type = key_type; + using compare_type = KeyCompare; + using value_compare = compare_type; + using reference = value_type&; + using const_reference = const value_type&; + using allocator_type = Allocator; + + static constexpr bool allow_multimapping = AllowMultimapping; + + static const key_type& get_key(const_reference val) { + return val; + } + + static value_compare value_comp(compare_type comp) { return comp; } +}; // struct set_traits + +template +class concurrent_multiset; + +template , typename Allocator = tbb::tbb_allocator> +class concurrent_set : public concurrent_skip_list, Allocator, false>> { + using base_type = concurrent_skip_list, Allocator, false>>; +public: + using key_type = Key; + using value_type = typename base_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Compare; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::const_pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + + using node_type = typename base_type::node_type; + + // Include constructors of base_type + using base_type::base_type; + + // Required for implicit deduction guides + concurrent_set() = default; + concurrent_set( const concurrent_set& ) = default; + concurrent_set( const concurrent_set& other, const allocator_type& alloc ) : base_type(other, alloc) {} + concurrent_set( concurrent_set&& ) = default; + concurrent_set( concurrent_set&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {} + // Required to respect the rule of 5 + concurrent_set& operator=( const concurrent_set& ) = default; + concurrent_set& operator=( concurrent_set&& ) = default; + + concurrent_set& operator=( std::initializer_list il ) { + base_type::operator= (il); + return *this; + } + + template + void merge(concurrent_set& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_set&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_multiset& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multiset&& source) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_set + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template >, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_set( It, It, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_set, Comp, Alloc>; + +template , + typename Alloc = tbb::tbb_allocator, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_set( std::initializer_list, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_set; + +template >, + typename = std::enable_if_t>> +concurrent_set( It, It, Alloc ) +-> concurrent_set, + std::less>, Alloc>; + +template >> +concurrent_set( std::initializer_list, Alloc ) +-> concurrent_set, Alloc>; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_set& lhs, + concurrent_set& rhs ) +{ + lhs.swap(rhs); +} + +template , typename Allocator = tbb::tbb_allocator> +class concurrent_multiset : public concurrent_skip_list, Allocator, true>> { + using base_type = concurrent_skip_list, Allocator, true>>; +public: + using key_type = Key; + using value_type = typename base_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Compare; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::const_pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + + using node_type = typename base_type::node_type; + + // Include constructors of base_type; + using base_type::base_type; + + // Required for implicit deduction guides + concurrent_multiset() = default; + concurrent_multiset( const concurrent_multiset& ) = default; + concurrent_multiset( const concurrent_multiset& other, const allocator_type& alloc ) : base_type(other, alloc) {} + concurrent_multiset( concurrent_multiset&& ) = default; + concurrent_multiset( concurrent_multiset&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {} + // Required to respect the rule of 5 + concurrent_multiset& operator=( const concurrent_multiset& ) = default; + concurrent_multiset& operator=( concurrent_multiset&& ) = default; + + concurrent_multiset& operator=( std::initializer_list il ) { + base_type::operator= (il); + return *this; + } + + template + void merge(concurrent_set& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_set&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_multiset& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multiset&& source) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_multiset + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template >, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_multiset( It, It, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_multiset, Comp, Alloc>; + +template , + typename Alloc = tbb::tbb_allocator, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_multiset( std::initializer_list, Comp = Comp(), Alloc = Alloc() ) +-> concurrent_multiset; + +template >, + typename = std::enable_if_t>> +concurrent_multiset( It, It, Alloc ) +-> concurrent_multiset, std::less>, Alloc>; + +template >> +concurrent_multiset( std::initializer_list, Alloc ) +-> concurrent_multiset, Alloc>; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_multiset& lhs, + concurrent_multiset& rhs ) +{ + lhs.swap(rhs); +} + +} // namespace d3 +} // namespace detail + +inline namespace v1 { + +using detail::d3::concurrent_set; +using detail::d3::concurrent_multiset; +using detail::split; + +} // inline namespace v1 +} // namespace tbb + +#endif // __TBB_concurrent_set_H diff --git a/src/3rd party/oneapi/tbb/concurrent_unordered_map.h b/src/3rd party/oneapi/tbb/concurrent_unordered_map.h new file mode 100644 index 0000000000..9cade0a94e --- /dev/null +++ b/src/3rd party/oneapi/tbb/concurrent_unordered_map.h @@ -0,0 +1,414 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_unordered_map_H +#define __TBB_concurrent_unordered_map_H + +#include "detail/_namespace_injection.h" +#include "detail/_concurrent_unordered_base.h" +#include "tbb_allocator.h" +#include + +namespace tbb { +namespace detail { +namespace d2 { + +template +struct concurrent_unordered_map_traits { + using value_type = std::pair; + using key_type = Key; + using allocator_type = Allocator; + using hash_compare_type = d1::hash_compare; + static constexpr bool allow_multimapping = AllowMultimapping; + + static constexpr const key_type& get_key( const value_type& value ) { + return value.first; + } +}; // struct concurrent_unordered_map_traits + +template +class concurrent_unordered_multimap; + +template , typename KeyEqual = std::equal_to, + typename Allocator = tbb::tbb_allocator> > +class concurrent_unordered_map + : public concurrent_unordered_base> +{ + using traits_type = concurrent_unordered_map_traits; + using base_type = concurrent_unordered_base; +public: + using key_type = typename base_type::key_type; + using mapped_type = T; + using value_type = typename base_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using hasher = typename base_type::hasher; + using key_equal = typename base_type::key_equal; + using allocator_type = typename base_type::allocator_type; + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::const_pointer; + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using local_iterator = typename base_type::local_iterator; + using const_local_iterator = typename base_type::const_local_iterator; + using node_type = typename base_type::node_type; + + // Include constructors of base type + using base_type::base_type; + + // Required for implicit deduction guides + concurrent_unordered_map() = default; + concurrent_unordered_map( const concurrent_unordered_map& ) = default; + concurrent_unordered_map( const concurrent_unordered_map& other, const allocator_type& alloc ) : base_type(other, alloc) {} + concurrent_unordered_map( concurrent_unordered_map&& ) = default; + concurrent_unordered_map( concurrent_unordered_map&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {} + // Required to respect the rule of 5 + concurrent_unordered_map& operator=( const concurrent_unordered_map& ) = default; + concurrent_unordered_map& operator=( concurrent_unordered_map&& ) = default; + + concurrent_unordered_map& operator=( std::initializer_list il ) { + base_type::operator= (il); + return *this; + } + + // Observers + mapped_type& operator[]( const key_type& key ) { + iterator where = this->find(key); + + if (where == this->end()) { + where = this->emplace(std::piecewise_construct, std::forward_as_tuple(key), std::tuple<>()).first; + } + return where->second; + } + + mapped_type& operator[]( key_type&& key ) { + iterator where = this->find(key); + + if (where == this->end()) { + where = this->emplace(std::piecewise_construct, std::forward_as_tuple(std::move(key)), std::tuple<>()).first; + } + return where->second; + } + + mapped_type& at( const key_type& key ) { + iterator where = this->find(key); + + if (where == this->end()) { + throw_exception(exception_id::invalid_key); + } + return where->second; + } + + const mapped_type& at( const key_type& key ) const { + const_iterator where = this->find(key); + + if (where == this->end()) { + throw_exception(exception_id::out_of_range); + } + return where->second; + } + + using base_type::insert; + + template + typename std::enable_if::value, + std::pair>::type insert( P&& value ) { + return this->emplace(std::forward

(value)); + } + + template + typename std::enable_if::value, + iterator>::type insert( const_iterator hint, P&& value ) { + return this->emplace_hint(hint, std::forward

(value)); + } + + template + void merge( concurrent_unordered_map& source ) { + this->internal_merge(source); + } + + template + void merge( concurrent_unordered_map&& source ) { + this->internal_merge(std::move(source)); + } + + template + void merge( concurrent_unordered_multimap& source ) { + this->internal_merge(source); + } + + template + void merge( concurrent_unordered_multimap&& source ) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_unordered_map + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +template >, + typename KeyEq = std::equal_to>, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_map( It, It, std::size_t = {}, + Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() ) +-> concurrent_unordered_map, iterator_mapped_t, Hash, KeyEq, Alloc>; + +template >, + typename KeyEq = std::equal_to>, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_map( std::initializer_list>, std::size_t = {}, + Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() ) +-> concurrent_unordered_map, T, Hash, KeyEq, Alloc>; + +template >, + typename = std::enable_if_t>> +concurrent_unordered_map( It, It, std::size_t, Alloc ) +-> concurrent_unordered_map, iterator_mapped_t, + std::hash>, + std::equal_to>, Alloc>; + +// TODO: investigate if a deduction guide for concurrent_unordered_map(It, It, Alloc) is needed + +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_map( It, It, std::size_t, Hash, Alloc ) +-> concurrent_unordered_map, iterator_mapped_t, + Hash, std::equal_to>, Alloc>; + +template >> +concurrent_unordered_map( std::initializer_list>, std::size_t, Alloc ) +-> concurrent_unordered_map, T, std::hash>, + std::equal_to>, Alloc>; + +template >> +concurrent_unordered_map( std::initializer_list>, Alloc ) +-> concurrent_unordered_map, T, std::hash>, + std::equal_to>, Alloc>; + +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_map( std::initializer_list>, std::size_t, Hash, Alloc ) +-> concurrent_unordered_map, T, Hash, + std::equal_to>, Alloc>; + +#if __APPLE__ && __TBB_CLANG_VERSION == 100000 +// An explicit deduction guide is required for copy/move constructor with allocator for APPLE LLVM 10.0.0 +// due to an issue with generating an implicit deduction guide for these constructors under several strange surcumstances. +// Currently the issue takes place because the last template parameter for Traits is boolean, it should not affect the deduction guides +// The issue reproduces only on this version of the compiler +template +concurrent_unordered_map( concurrent_unordered_map, Alloc ) +-> concurrent_unordered_map; +#endif + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_unordered_map& lhs, + concurrent_unordered_map& rhs ) { + lhs.swap(rhs); +} + +template , typename KeyEqual = std::equal_to, + typename Allocator = tbb::tbb_allocator> > +class concurrent_unordered_multimap + : public concurrent_unordered_base> +{ + using traits_type = concurrent_unordered_map_traits; + using base_type = concurrent_unordered_base; +public: + using key_type = typename base_type::key_type; + using mapped_type = T; + using value_type = typename base_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using hasher = typename base_type::hasher; + using key_equal = typename base_type::key_equal; + using allocator_type = typename base_type::allocator_type; + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::const_pointer; + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using local_iterator = typename base_type::local_iterator; + using const_local_iterator = typename base_type::const_local_iterator; + using node_type = typename base_type::node_type; + + // Include constructors of base type + using base_type::base_type; + using base_type::insert; + + // Required for implicit deduction guides + concurrent_unordered_multimap() = default; + concurrent_unordered_multimap( const concurrent_unordered_multimap& ) = default; + concurrent_unordered_multimap( const concurrent_unordered_multimap& other, const allocator_type& alloc ) : base_type(other, alloc) {} + concurrent_unordered_multimap( concurrent_unordered_multimap&& ) = default; + concurrent_unordered_multimap( concurrent_unordered_multimap&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {} + // Required to respect the rule of 5 + concurrent_unordered_multimap& operator=( const concurrent_unordered_multimap& ) = default; + concurrent_unordered_multimap& operator=( concurrent_unordered_multimap&& ) = default; + + concurrent_unordered_multimap& operator=( std::initializer_list il ) { + base_type::operator= (il); + return *this; + } + + template + typename std::enable_if::value, + std::pair>::type insert( P&& value ) { + return this->emplace(std::forward

(value)); + } + + template + typename std::enable_if::value, + iterator>::type insert( const_iterator hint, P&& value ) { + return this->emplace_hint(hint, std::forward(value)); + } + + template + void merge( concurrent_unordered_map& source ) { + this->internal_merge(source); + } + + template + void merge( concurrent_unordered_map&& source ) { + this->internal_merge(std::move(source)); + } + + template + void merge( concurrent_unordered_multimap& source ) { + this->internal_merge(source); + } + + template + void merge( concurrent_unordered_multimap&& source ) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_unordered_multimap + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template >, + typename KeyEq = std::equal_to>, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_multimap( It, It, std::size_t = {}, Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() ) +-> concurrent_unordered_multimap, iterator_mapped_t, Hash, KeyEq, Alloc>; + +template >, + typename KeyEq = std::equal_to>, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_multimap( std::initializer_list>, std::size_t = {}, + Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() ) +-> concurrent_unordered_multimap, T, Hash, KeyEq, Alloc>; + +template >, + typename = std::enable_if_t>> +concurrent_unordered_multimap( It, It, std::size_t, Alloc ) +-> concurrent_unordered_multimap, iterator_mapped_t, + std::hash>, + std::equal_to>, Alloc>; + +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_multimap( It, It, std::size_t, Hash, Alloc ) +-> concurrent_unordered_multimap, iterator_mapped_t, Hash, + std::equal_to>, Alloc>; + +template >> +concurrent_unordered_multimap( std::initializer_list>, std::size_t, Alloc ) +-> concurrent_unordered_multimap, T, std::hash>, + std::equal_to>, Alloc>; + +template >> +concurrent_unordered_multimap( std::initializer_list>, Alloc ) +-> concurrent_unordered_multimap, T, std::hash>, + std::equal_to>, Alloc>; + +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_multimap( std::initializer_list>, std::size_t, Hash, Alloc ) +-> concurrent_unordered_multimap, T, Hash, + std::equal_to>, Alloc>; + +#if __APPLE__ && __TBB_CLANG_VERSION == 100000 +// An explicit deduction guide is required for copy/move constructor with allocator for APPLE LLVM 10.0.0 +// due to an issue with generating an implicit deduction guide for these constructors under several strange surcumstances. +// Currently the issue takes place because the last template parameter for Traits is boolean, it should not affect the deduction guides +// The issue reproduces only on this version of the compiler +template +concurrent_unordered_multimap( concurrent_unordered_multimap, Alloc ) +-> concurrent_unordered_multimap; +#endif +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_unordered_multimap& lhs, + concurrent_unordered_multimap& rhs ) { + lhs.swap(rhs); +} + +} // namespace d2 +} // namespace detail + +inline namespace v1 { + +using detail::d2::concurrent_unordered_map; +using detail::d2::concurrent_unordered_multimap; +using detail::split; + +} // inline namespace v1 +} // namespace tbb + +#endif // __TBB_concurrent_unordered_map_H diff --git a/src/3rd party/oneapi/tbb/concurrent_unordered_set.h b/src/3rd party/oneapi/tbb/concurrent_unordered_set.h new file mode 100644 index 0000000000..b7e4b4cafc --- /dev/null +++ b/src/3rd party/oneapi/tbb/concurrent_unordered_set.h @@ -0,0 +1,333 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_unordered_set_H +#define __TBB_concurrent_unordered_set_H + +#include "detail/_namespace_injection.h" +#include "detail/_concurrent_unordered_base.h" +#include "tbb_allocator.h" + +namespace tbb { +namespace detail { +namespace d2 { + +template +struct concurrent_unordered_set_traits { + using key_type = Key; + using value_type = key_type; + using allocator_type = Allocator; + using hash_compare_type = d1::hash_compare; + static constexpr bool allow_multimapping = AllowMultimapping; + + static constexpr const key_type& get_key( const value_type& value ) { + return value; + } +}; // class concurrent_unordered_set_traits + +template +class concurrent_unordered_multiset; + +template , typename KeyEqual = std::equal_to, + typename Allocator = tbb::tbb_allocator> +class concurrent_unordered_set + : public concurrent_unordered_base> +{ + using traits_type = concurrent_unordered_set_traits; + using base_type = concurrent_unordered_base; +public: + using key_type = typename base_type::key_type; + using value_type = typename base_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using hasher = typename base_type::hasher; + using key_equal = typename base_type::key_equal; + using allocator_type = typename base_type::allocator_type; + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::const_pointer; + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using local_iterator = typename base_type::local_iterator; + using const_local_iterator = typename base_type::const_local_iterator; + using node_type = typename base_type::node_type; + + // Include constructors of base_type; + using base_type::base_type; + + // Required for implicit deduction guides + concurrent_unordered_set() = default; + concurrent_unordered_set( const concurrent_unordered_set& ) = default; + concurrent_unordered_set( const concurrent_unordered_set& other, const allocator_type& alloc ) : base_type(other, alloc) {} + concurrent_unordered_set( concurrent_unordered_set&& ) = default; + concurrent_unordered_set( concurrent_unordered_set&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {} + // Required to respect the rule of 5 + concurrent_unordered_set& operator=( const concurrent_unordered_set& ) = default; + concurrent_unordered_set& operator=( concurrent_unordered_set&& ) = default; + + concurrent_unordered_set& operator=( std::initializer_list il ) { + base_type::operator= (il); + return *this; + } + + template + void merge( concurrent_unordered_set& source ) { + this->internal_merge(source); + } + + template + void merge( concurrent_unordered_set&& source ) { + this->internal_merge(std::move(source)); + } + + template + void merge( concurrent_unordered_multiset& source ) { + this->internal_merge(source); + } + + template + void merge( concurrent_unordered_multiset&& source ) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_unordered_set + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template >, + typename KeyEq = std::equal_to>, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_set( It, It, std::size_t = {}, Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() ) +-> concurrent_unordered_set, Hash, KeyEq, Alloc>; + +template , + typename KeyEq = std::equal_to, + typename Alloc = tbb::tbb_allocator, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_set( std::initializer_list, std::size_t = {}, + Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() ) +-> concurrent_unordered_set; + +template >, + typename = std::enable_if_t>> +concurrent_unordered_set( It, It, std::size_t, Alloc ) +-> concurrent_unordered_set, std::hash>, + std::equal_to>, Alloc>; + +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_set( It, It, std::size_t, Hash, Alloc ) +-> concurrent_unordered_set, Hash, std::equal_to>, Alloc>; + +template >> +concurrent_unordered_set( std::initializer_list, std::size_t, Alloc ) +-> concurrent_unordered_set, std::equal_to, Alloc>; + +template >> +concurrent_unordered_set( std::initializer_list, Alloc ) +-> concurrent_unordered_set, std::equal_to, Alloc>; + +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_set( std::initializer_list, std::size_t, Hash, Alloc ) +-> concurrent_unordered_set, Alloc>; + +#if __APPLE__ && __TBB_CLANG_VERSION == 100000 +// An explicit deduction guide is required for copy/move constructor with allocator for APPLE LLVM 10.0.0 +// due to an issue with generating an implicit deduction guide for these constructors under several strange surcumstances. +// Currently the issue takes place because the last template parameter for Traits is boolean, it should not affect the deduction guides +// The issue reproduces only on this version of the compiler +template +concurrent_unordered_set( concurrent_unordered_set, Alloc ) +-> concurrent_unordered_set; +#endif +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_unordered_set& lhs, + concurrent_unordered_set& rhs ) { + lhs.swap(rhs); +} + +template , typename KeyEqual = std::equal_to, + typename Allocator = tbb::tbb_allocator> +class concurrent_unordered_multiset + : public concurrent_unordered_base> +{ + using traits_type = concurrent_unordered_set_traits; + using base_type = concurrent_unordered_base; +public: + using key_type = typename base_type::key_type; + using value_type = typename base_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using hasher = typename base_type::hasher; + using key_equal = typename base_type::key_equal; + using allocator_type = typename base_type::allocator_type; + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::const_pointer; + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using local_iterator = typename base_type::local_iterator; + using const_local_iterator = typename base_type::const_local_iterator; + using node_type = typename base_type::node_type; + + // Include constructors of base_type; + using base_type::base_type; + + // Required for implicit deduction guides + concurrent_unordered_multiset() = default; + concurrent_unordered_multiset( const concurrent_unordered_multiset& ) = default; + concurrent_unordered_multiset( const concurrent_unordered_multiset& other, const allocator_type& alloc ) : base_type(other, alloc) {} + concurrent_unordered_multiset( concurrent_unordered_multiset&& ) = default; + concurrent_unordered_multiset( concurrent_unordered_multiset&& other, const allocator_type& alloc ) : base_type(std::move(other), alloc) {} + // Required to respect the rule of 5 + concurrent_unordered_multiset& operator=( const concurrent_unordered_multiset& ) = default; + concurrent_unordered_multiset& operator=( concurrent_unordered_multiset&& ) = default; + + concurrent_unordered_multiset& operator=( std::initializer_list il ) { + base_type::operator= (il); + return *this; + } + + template + void merge( concurrent_unordered_set& source ) { + this->internal_merge(source); + } + + template + void merge( concurrent_unordered_set&& source ) { + this->internal_merge(std::move(source)); + } + + template + void merge( concurrent_unordered_multiset& source ) { + this->internal_merge(source); + } + + template + void merge( concurrent_unordered_multiset&& source ) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_unordered_multiset + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +template >, + typename KeyEq = std::equal_to>, + typename Alloc = tbb::tbb_allocator>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_multiset( It, It, std::size_t = {}, Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() ) +-> concurrent_unordered_multiset, Hash, KeyEq, Alloc>; + +template , + typename KeyEq = std::equal_to, + typename Alloc = tbb::tbb_allocator, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_multiset( std::initializer_list, std::size_t = {}, + Hash = Hash(), KeyEq = KeyEq(), Alloc = Alloc() ) +-> concurrent_unordered_multiset; + +template >, + typename = std::enable_if_t>> +concurrent_unordered_multiset( It, It, std::size_t, Alloc ) +-> concurrent_unordered_multiset, std::hash>, + std::equal_to>, Alloc>; + +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_multiset( It, It, std::size_t, Hash, Alloc ) +-> concurrent_unordered_multiset, Hash, std::equal_to>, Alloc>; + +template >> +concurrent_unordered_multiset( std::initializer_list, std::size_t, Alloc ) +-> concurrent_unordered_multiset, std::equal_to, Alloc>; + +template >> +concurrent_unordered_multiset( std::initializer_list, Alloc ) +-> concurrent_unordered_multiset, std::equal_to, Alloc>; + +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_unordered_multiset( std::initializer_list, std::size_t, Hash, Alloc ) +-> concurrent_unordered_multiset, Alloc>; + +#if __APPLE__ && __TBB_CLANG_VERSION == 100000 +// An explicit deduction guide is required for copy/move constructor with allocator for APPLE LLVM 10.0.0 +// due to an issue with generating an implicit deduction guide for these constructors under several strange surcumstances. +// Currently the issue takes place because the last template parameter for Traits is boolean, it should not affect the deduction guides +// The issue reproduces only on this version of the compiler +template +concurrent_unordered_multiset( concurrent_unordered_multiset, Alloc ) +-> concurrent_unordered_multiset; +#endif +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +void swap( concurrent_unordered_multiset& lhs, + concurrent_unordered_multiset& rhs ) { + lhs.swap(rhs); +} + +} // namespace d2 +} // namespace detail + +inline namespace v1 { + +using detail::d2::concurrent_unordered_set; +using detail::d2::concurrent_unordered_multiset; +using detail::split; + +} // inline namespace v1 +} // namespace tbb + +#endif // __TBB_concurrent_unordered_set_H diff --git a/src/3rd party/oneapi/tbb/concurrent_vector.h b/src/3rd party/oneapi/tbb/concurrent_vector.h new file mode 100644 index 0000000000..2649459003 --- /dev/null +++ b/src/3rd party/oneapi/tbb/concurrent_vector.h @@ -0,0 +1,1130 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_vector_H +#define __TBB_concurrent_vector_H + +#include "detail/_namespace_injection.h" +#include "detail/_utils.h" +#include "detail/_assert.h" +#include "detail/_allocator_traits.h" +#include "detail/_segment_table.h" +#include "detail/_containers_helpers.h" +#include "blocked_range.h" +#include "cache_aligned_allocator.h" + +#include +#include // std::move_if_noexcept +#include +#if __TBB_CPP20_COMPARISONS_PRESENT +#include +#endif + +namespace tbb { +namespace detail { +namespace d1 { + +template +class vector_iterator { + using vector_type = Vector; + +public: + using value_type = Value; + using size_type = typename vector_type::size_type; + using difference_type = typename vector_type::difference_type; + using pointer = value_type*; + using reference = value_type&; + using iterator_category = std::random_access_iterator_tag; + + template + friend vector_iterator operator+( typename vector_iterator::difference_type, const vector_iterator& ); + + template + friend typename vector_iterator::difference_type operator-( const vector_iterator&, const vector_iterator& ); + + template + friend bool operator==( const vector_iterator&, const vector_iterator& ); + + template + friend bool operator<( const vector_iterator&, const vector_iterator& ); + + template + friend class vector_iterator; + + template + friend class concurrent_vector; + +private: + vector_iterator( const vector_type& vector, size_type index, value_type* item = nullptr ) + : my_vector(const_cast(&vector)), my_index(index), my_item(item) + {} + +public: + vector_iterator() : my_vector(nullptr), my_index(~size_type(0)), my_item(nullptr) + {} + + vector_iterator( const vector_iterator& other ) + : my_vector(other.my_vector), my_index(other.my_index), my_item(other.my_item) + {} + + vector_iterator& operator=( const vector_iterator& other ) { + my_vector = other.my_vector; + my_index = other.my_index; + my_item = other.my_item; + return *this; + } + + vector_iterator operator+( difference_type offset ) const { + return vector_iterator(*my_vector, my_index + offset); + } + + vector_iterator& operator+=( difference_type offset ) { + my_index += offset; + my_item = nullptr; + return *this; + } + + vector_iterator operator-( difference_type offset ) const { + return vector_iterator(*my_vector, my_index - offset); + } + + vector_iterator& operator-=( difference_type offset ) { + my_index -= offset; + my_item = nullptr; + return *this; + } + + reference operator*() const { + value_type *item = my_item; + if (item == nullptr) { + item = &my_vector->internal_subscript(my_index); + } else { + __TBB_ASSERT(item == &my_vector->internal_subscript(my_index), "corrupt cache"); + } + return *item; + } + + pointer operator->() const { return &(operator*()); } + + reference operator[]( difference_type k ) const { + return my_vector->internal_subscript(my_index + k); + } + + vector_iterator& operator++() { + ++my_index; + if (my_item != nullptr) { + if (vector_type::is_first_element_in_segment(my_index)) { + // If the iterator crosses a segment boundary, the pointer become invalid + // as possibly next segment is in another memory location + my_item = nullptr; + } else { + ++my_item; + } + } + return *this; + } + + vector_iterator operator++(int) { + vector_iterator result = *this; + ++(*this); + return result; + } + + vector_iterator& operator--() { + __TBB_ASSERT(my_index > 0, "operator--() applied to iterator already at beginning of concurrent_vector"); + --my_index; + if (my_item != nullptr) { + if (vector_type::is_first_element_in_segment(my_index)) { + // If the iterator crosses a segment boundary, the pointer become invalid + // as possibly next segment is in another memory location + my_item = nullptr; + } else { + --my_item; + } + } + return *this; + } + + vector_iterator operator--(int) { + vector_iterator result = *this; + --(*this); + return result; + } + +private: + // concurrent_vector over which we are iterating. + vector_type* my_vector; + + // Index into the vector + size_type my_index; + + // Caches my_vector *it; + // If my_item == nullptr cached value is not available use internal_subscript(my_index) + mutable value_type* my_item; +}; // class vector_iterator + +template +vector_iterator operator+( typename vector_iterator::difference_type offset, + const vector_iterator& v ) +{ + return vector_iterator(*v.my_vector, v.my_index + offset); +} + +template +typename vector_iterator::difference_type operator-( const vector_iterator& i, + const vector_iterator& j ) +{ + using difference_type = typename vector_iterator::difference_type; + return static_cast(i.my_index) - static_cast(j.my_index); +} + +template +bool operator==( const vector_iterator& i, const vector_iterator& j ) { + return i.my_vector == j.my_vector && i.my_index == j.my_index; +} + +template +bool operator!=( const vector_iterator& i, const vector_iterator& j ) { + return !(i == j); +} + +template +bool operator<( const vector_iterator& i, const vector_iterator& j ) { + return i.my_index < j.my_index; +} + +template +bool operator>( const vector_iterator& i, const vector_iterator& j ) { + return j < i; +} + +template +bool operator>=( const vector_iterator& i, const vector_iterator& j ) { + return !(i < j); +} + +template +bool operator<=( const vector_iterator& i, const vector_iterator& j ) { + return !(j < i); +} + +__TBB_GLOBAL_VAR constexpr std::size_t embedded_table_num_segments = 3; + +template > +class concurrent_vector + : private segment_table, embedded_table_num_segments> +{ + using self_type = concurrent_vector; + using base_type = segment_table; + + friend class segment_table; + + template + class generic_range_type : public tbb::blocked_range { + using base_type = tbb::blocked_range; + public: + using value_type = T; + using reference = T&; + using const_reference = const T&; + using iterator = Iterator; + using difference_type = std::ptrdiff_t; + + using base_type::base_type; + + template + generic_range_type( const generic_range_type& r) : blocked_range(r.begin(), r.end(), r.grainsize()) {} + generic_range_type( generic_range_type& r, split ) : blocked_range(r, split()) {} + }; // class generic_range_type + + static_assert(std::is_same::value, + "value_type of the container must be the same as its allocator's"); + using allocator_traits_type = tbb::detail::allocator_traits; + // Segment table for concurrent_vector can be extended + static constexpr bool allow_table_extending = true; + static constexpr bool is_noexcept_assignment = allocator_traits_type::propagate_on_container_move_assignment::value || + allocator_traits_type::is_always_equal::value; + static constexpr bool is_noexcept_swap = allocator_traits_type::propagate_on_container_swap::value || + allocator_traits_type::is_always_equal::value; + +public: + using value_type = T; + using allocator_type = Allocator; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using reference = value_type&; + using const_reference = const value_type&; + + using pointer = typename allocator_traits_type::pointer; + using const_pointer = typename allocator_traits_type::const_pointer; + + using iterator = vector_iterator; + using const_iterator = vector_iterator; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + + using range_type = generic_range_type; + using const_range_type = generic_range_type; + + concurrent_vector() : concurrent_vector(allocator_type()) {} + + explicit concurrent_vector( const allocator_type& alloc ) noexcept + : base_type(alloc) + {} + + explicit concurrent_vector( size_type count, const value_type& value, + const allocator_type& alloc = allocator_type() ) + : concurrent_vector(alloc) + { + try_call( [&] { + grow_by(count, value); + } ).on_exception( [&] { + base_type::clear(); + }); + } + + explicit concurrent_vector( size_type count, const allocator_type& alloc = allocator_type() ) + : concurrent_vector(alloc) + { + try_call( [&] { + grow_by(count); + } ).on_exception( [&] { + base_type::clear(); + }); + } + + template + concurrent_vector( InputIterator first, InputIterator last, const allocator_type& alloc = allocator_type() ) + : concurrent_vector(alloc) + { + try_call( [&] { + grow_by(first, last); + } ).on_exception( [&] { + base_type::clear(); + }); + } + + concurrent_vector( const concurrent_vector& other ) + : base_type(segment_table_allocator_traits::select_on_container_copy_construction(other.get_allocator())) + { + try_call( [&] { + grow_by(other.begin(), other.end()); + } ).on_exception( [&] { + base_type::clear(); + }); + } + + concurrent_vector( const concurrent_vector& other, const allocator_type& alloc ) + : base_type(other, alloc) {} + + concurrent_vector(concurrent_vector&& other) noexcept + : base_type(std::move(other)) + {} + + concurrent_vector( concurrent_vector&& other, const allocator_type& alloc ) + : base_type(std::move(other), alloc) + {} + + concurrent_vector( std::initializer_list init, + const allocator_type& alloc = allocator_type() ) + : concurrent_vector(init.begin(), init.end(), alloc) + {} + + ~concurrent_vector() {} + + // Assignment + concurrent_vector& operator=( const concurrent_vector& other ) { + base_type::operator=(other); + return *this; + } + + concurrent_vector& operator=( concurrent_vector&& other ) noexcept(is_noexcept_assignment) { + base_type::operator=(std::move(other)); + return *this; + } + + concurrent_vector& operator=( std::initializer_list init ) { + assign(init); + return *this; + } + + void assign( size_type count, const value_type& value ) { + destroy_elements(); + grow_by(count, value); + } + + template + typename std::enable_if::value, void>::type + assign( InputIterator first, InputIterator last ) { + destroy_elements(); + grow_by(first, last); + } + + void assign( std::initializer_list init ) { + destroy_elements(); + assign(init.begin(), init.end()); + } + + // Concurrent growth + iterator grow_by( size_type delta ) { + return internal_grow_by_delta(delta); + } + + iterator grow_by( size_type delta, const value_type& value ) { + return internal_grow_by_delta(delta, value); + } + + template + typename std::enable_if::value, iterator>::type + grow_by( ForwardIterator first, ForwardIterator last ) { + auto delta = std::distance(first, last); + return internal_grow_by_delta(delta, first, last); + } + + iterator grow_by( std::initializer_list init ) { + return grow_by(init.begin(), init.end()); + } + + iterator grow_to_at_least( size_type n ) { + return internal_grow_to_at_least(n); + } + iterator grow_to_at_least( size_type n, const value_type& value ) { + return internal_grow_to_at_least(n, value); + } + + iterator push_back( const value_type& item ) { + return internal_emplace_back(item); + } + + iterator push_back( value_type&& item ) { + return internal_emplace_back(std::move(item)); + } + + template + iterator emplace_back( Args&&... args ) { + return internal_emplace_back(std::forward(args)...); + } + + // Items access + reference operator[]( size_type index ) { + return internal_subscript(index); + } + const_reference operator[]( size_type index ) const { + return internal_subscript(index); + } + + reference at( size_type index ) { + return internal_subscript_with_exceptions(index); + } + const_reference at( size_type index ) const { + return internal_subscript_with_exceptions(index); + } + + // Get range for iterating with parallel algorithms + range_type range( size_t grainsize = 1 ) { + return range_type(begin(), end(), grainsize); + } + + // Get const range for iterating with parallel algorithms + const_range_type range( size_t grainsize = 1 ) const { + return const_range_type(begin(), end(), grainsize); + } + + reference front() { + return internal_subscript(0); + } + + const_reference front() const { + return internal_subscript(0); + } + + reference back() { + return internal_subscript(size() - 1); + } + + const_reference back() const { + return internal_subscript(size() - 1); + } + + // Iterators + iterator begin() { return iterator(*this, 0); } + const_iterator begin() const { return const_iterator(*this, 0); } + const_iterator cbegin() const { return const_iterator(*this, 0); } + + iterator end() { return iterator(*this, size()); } + const_iterator end() const { return const_iterator(*this, size()); } + const_iterator cend() const { return const_iterator(*this, size()); } + + reverse_iterator rbegin() { return reverse_iterator(end()); } + const_reverse_iterator rbegin() const { return const_reverse_iterator(end()); } + const_reverse_iterator crbegin() const { return const_reverse_iterator(cend()); } + + reverse_iterator rend() { return reverse_iterator(begin()); } + const_reverse_iterator rend() const { return const_reverse_iterator(begin()); } + const_reverse_iterator crend() const { return const_reverse_iterator(cbegin()); } + + allocator_type get_allocator() const { + return base_type::get_allocator(); + } + + // Storage + bool empty() const noexcept { + return 0 == size(); + } + + size_type size() const noexcept { + return std::min(this->my_size.load(std::memory_order_acquire), capacity()); + } + + size_type max_size() const noexcept { + return allocator_traits_type::max_size(base_type::get_allocator()); + } + + size_type capacity() const noexcept { + return base_type::capacity(); + } + + void reserve( size_type n ) { + if (n == 0) return; + + if (n > max_size()) { + tbb::detail::throw_exception(exception_id::reservation_length_error); + } + + this->assign_first_block_if_necessary(this->segment_index_of(n - 1) + 1); + base_type::reserve(n); + } + + void resize( size_type n ) { + internal_resize(n); + } + + void resize( size_type n, const value_type& val ) { + internal_resize(n, val); + } + + void shrink_to_fit() { + internal_compact(); + } + + void swap(concurrent_vector& other) noexcept(is_noexcept_swap) { + base_type::swap(other); + } + + void clear() { + destroy_elements(); + } + +private: + using segment_type = typename base_type::segment_type; + using segment_table_type = typename base_type::segment_table_type; + using segment_table_allocator_traits = typename base_type::segment_table_allocator_traits; + using segment_index_type = typename base_type::segment_index_type; + + using segment_element_type = typename base_type::value_type; + using segment_element_allocator_type = typename allocator_traits_type::template rebind_alloc; + using segment_element_allocator_traits = tbb::detail::allocator_traits; + + segment_table_type allocate_long_table( const typename base_type::atomic_segment* embedded_table, size_type start_index ) { + __TBB_ASSERT(start_index <= this->embedded_table_size, "Start index out of embedded table"); + + // If other threads are trying to set pointers in the short segment, wait for them to finish their + // assignments before we copy the short segment to the long segment. Note: grow_to_at_least depends on it + for (segment_index_type i = 0; this->segment_base(i) < start_index; ++i) { + spin_wait_while_eq(embedded_table[i], segment_type(nullptr)); + } + + // It is possible that the table was extend by a thread allocating first_block, need to check this. + if (this->get_table() != embedded_table) { + return nullptr; + } + + // Allocate long segment table and fill with null pointers + segment_table_type new_segment_table = segment_table_allocator_traits::allocate(base_type::get_allocator(), this->pointers_per_long_table); + // Copy segment pointers from the embedded table + for (size_type segment_index = 0; segment_index < this->pointers_per_embedded_table; ++segment_index) { + segment_table_allocator_traits::construct(base_type::get_allocator(), &new_segment_table[segment_index], + embedded_table[segment_index].load(std::memory_order_relaxed)); + } + for (size_type segment_index = this->pointers_per_embedded_table; segment_index < this->pointers_per_long_table; ++segment_index) { + segment_table_allocator_traits::construct(base_type::get_allocator(), &new_segment_table[segment_index], nullptr); + } + + return new_segment_table; + } + + // create_segment function is required by the segment_table base class + segment_type create_segment( segment_table_type table, segment_index_type seg_index, size_type index ) { + size_type first_block = this->my_first_block.load(std::memory_order_relaxed); + // First block allocation + if (seg_index < first_block) { + // If 0 segment is already allocated, then it remains to wait until the segments are filled to requested + if (table[0].load(std::memory_order_acquire) != nullptr) { + spin_wait_while_eq(table[seg_index], segment_type(nullptr)); + return nullptr; + } + + segment_element_allocator_type segment_allocator(base_type::get_allocator()); + segment_type new_segment = nullptr; + size_type first_block_size = this->segment_size(first_block); + try_call( [&] { + new_segment = segment_element_allocator_traits::allocate(segment_allocator, first_block_size); + } ).on_exception( [&] { + segment_type disabled_segment = nullptr; + if (table[0].compare_exchange_strong(disabled_segment, this->segment_allocation_failure_tag)) { + size_type end_segment = table == this->my_embedded_table ? this->pointers_per_embedded_table : first_block; + for (size_type i = 1; i < end_segment; ++i) { + table[i].store(this->segment_allocation_failure_tag, std::memory_order_release); + } + } + }); + + segment_type disabled_segment = nullptr; + if (table[0].compare_exchange_strong(disabled_segment, new_segment)) { + this->extend_table_if_necessary(table, /*start_index*/0, /*end_index*/first_block_size); + for (size_type i = 1; i < first_block; ++i) { + table[i].store(new_segment, std::memory_order_release); + } + + // Other threads can wait on a snapshot of an embedded table, need to fill it. + for (size_type i = 1; i < first_block && i < this->pointers_per_embedded_table; ++i) { + this->my_embedded_table[i].store(new_segment, std::memory_order_release); + } + } else if (new_segment != this->segment_allocation_failure_tag) { + // Deallocate the memory + segment_element_allocator_traits::deallocate(segment_allocator, new_segment, first_block_size); + // 0 segment is already allocated, then it remains to wait until the segments are filled to requested + spin_wait_while_eq(table[seg_index], segment_type(nullptr)); + } + } else { + size_type offset = this->segment_base(seg_index); + if (index == offset) { + __TBB_ASSERT(table[seg_index].load(std::memory_order_relaxed) == nullptr, "Only this thread can enable this segment"); + segment_element_allocator_type segment_allocator(base_type::get_allocator()); + segment_type new_segment = this->segment_allocation_failure_tag; + try_call( [&] { + new_segment = segment_element_allocator_traits::allocate(segment_allocator,this->segment_size(seg_index)); + // Shift base address to simplify access by index + new_segment -= this->segment_base(seg_index); + } ).on_completion( [&] { + table[seg_index].store(new_segment, std::memory_order_release); + }); + } else { + spin_wait_while_eq(table[seg_index], segment_type(nullptr)); + } + } + return nullptr; + } + + // Returns the number of elements in the segment to be destroy + size_type number_of_elements_in_segment( segment_index_type seg_index ) { + size_type curr_vector_size = this->my_size.load(std::memory_order_relaxed); + size_type curr_segment_base = this->segment_base(seg_index); + + if (seg_index == 0) { + return std::min(curr_vector_size, this->segment_size(seg_index)); + } else { + // Perhaps the segment is allocated, but there are no elements in it. + if (curr_vector_size < curr_segment_base) { + return 0; + } + return curr_segment_base * 2 > curr_vector_size ? curr_vector_size - curr_segment_base : curr_segment_base; + } + } + + segment_type nullify_segment( segment_table_type table, size_type segment_index ) { + segment_type target_segment = table[segment_index].load(std::memory_order_relaxed); + if (segment_index >= this->my_first_block) { + table[segment_index].store(nullptr, std::memory_order_relaxed); + } else { + if (segment_index == 0) { + for (size_type i = 0; i < this->my_first_block; ++i) { + table[i].store(nullptr, std::memory_order_relaxed); + } + } + } + + return target_segment; + } + + void deallocate_segment( segment_type address, segment_index_type seg_index ) { + segment_element_allocator_type segment_allocator(base_type::get_allocator()); + size_type first_block = this->my_first_block.load(std::memory_order_relaxed); + if (seg_index >= first_block) { + segment_element_allocator_traits::deallocate(segment_allocator, address, this->segment_size(seg_index)); + } + else if (seg_index == 0) { + size_type elements_to_deallocate = first_block > 0 ? this->segment_size(first_block) : this->segment_size(0); + segment_element_allocator_traits::deallocate(segment_allocator, address, elements_to_deallocate); + } + } + + // destroy_segment function is required by the segment_table base class + void destroy_segment( segment_type address, segment_index_type seg_index ) { + size_type elements_to_destroy = number_of_elements_in_segment(seg_index); + segment_element_allocator_type segment_allocator(base_type::get_allocator()); + + for (size_type i = 0; i < elements_to_destroy; ++i) { + segment_element_allocator_traits::destroy(segment_allocator, address + i); + } + + deallocate_segment(address, seg_index); + } + + // copy_segment function is required by the segment_table base class + void copy_segment( segment_index_type seg_index, segment_type from, segment_type to ) { + size_type i = 0; + try_call( [&] { + for (; i != number_of_elements_in_segment(seg_index); ++i) { + segment_table_allocator_traits::construct(base_type::get_allocator(), to + i, from[i]); + } + } ).on_exception( [&] { + // Zero-initialize items left not constructed after the exception + zero_unconstructed_elements(this->get_segment(seg_index) + i, this->segment_size(seg_index) - i); + + segment_index_type last_segment = this->segment_index_of(this->my_size.load(std::memory_order_relaxed)); + auto table = this->get_table(); + for (segment_index_type j = seg_index + 1; j != last_segment; ++j) { + auto curr_segment = table[j].load(std::memory_order_relaxed); + if (curr_segment) { + zero_unconstructed_elements(curr_segment + this->segment_base(j), this->segment_size(j)); + } + } + this->my_size.store(this->segment_size(seg_index) + i, std::memory_order_relaxed); + }); + } + + // move_segment function is required by the segment_table base class + void move_segment( segment_index_type seg_index, segment_type from, segment_type to ) { + size_type i = 0; + try_call( [&] { + for (; i != number_of_elements_in_segment(seg_index); ++i) { + segment_table_allocator_traits::construct(base_type::get_allocator(), to + i, std::move(from[i])); + } + } ).on_exception( [&] { + // Zero-initialize items left not constructed after the exception + zero_unconstructed_elements(this->get_segment(seg_index) + i, this->segment_size(seg_index) - i); + + segment_index_type last_segment = this->segment_index_of(this->my_size.load(std::memory_order_relaxed)); + auto table = this->get_table(); + for (segment_index_type j = seg_index + 1; j != last_segment; ++j) { + auto curr_segment = table[j].load(std::memory_order_relaxed); + if (curr_segment) { + zero_unconstructed_elements(curr_segment + this->segment_base(j), this->segment_size(j)); + } + } + this->my_size.store(this->segment_size(seg_index) + i, std::memory_order_relaxed); + }); + } + + static constexpr bool is_first_element_in_segment( size_type index ) { + // An element is the first in a segment if its index is equal to a power of two + return is_power_of_two_at_least(index, 2); + } + + const_reference internal_subscript( size_type index ) const { + return const_cast(this)->internal_subscript(index); + } + + reference internal_subscript( size_type index ) { + __TBB_ASSERT(index < this->my_size.load(std::memory_order_relaxed), "Invalid subscript index"); + return base_type::template internal_subscript(index); + } + + const_reference internal_subscript_with_exceptions( size_type index ) const { + return const_cast(this)->internal_subscript_with_exceptions(index); + } + + reference internal_subscript_with_exceptions( size_type index ) { + if (index >= this->my_size.load(std::memory_order_acquire)) { + tbb::detail::throw_exception(exception_id::out_of_range); + } + + segment_table_type table = this->my_segment_table.load(std::memory_order_acquire); + + size_type seg_index = this->segment_index_of(index); + if (base_type::number_of_segments(table) < seg_index) { + tbb::detail::throw_exception(exception_id::out_of_range); + } + + if (table[seg_index] <= this->segment_allocation_failure_tag) { + tbb::detail::throw_exception(exception_id::out_of_range); + } + + return base_type::template internal_subscript(index); + } + + static void zero_unconstructed_elements( pointer start, size_type count ) { + std::memset(static_cast(start), 0, count * sizeof(value_type)); + } + + template + iterator internal_emplace_back( Args&&... args ) { + size_type old_size = this->my_size++; + this->assign_first_block_if_necessary(default_first_block_size); + auto element_address = &base_type::template internal_subscript(old_size); + + // try_call API is not convenient here due to broken + // variadic capture on GCC 4.8.5 + auto value_guard = make_raii_guard([&] { + zero_unconstructed_elements(element_address, /*count =*/1); + }); + + segment_table_allocator_traits::construct(base_type::get_allocator(), element_address, std::forward(args)...); + value_guard.dismiss(); + return iterator(*this, old_size, element_address); + } + + template + void internal_loop_construct( segment_table_type table, size_type start_idx, size_type end_idx, const Args&... args ) { + static_assert(sizeof...(Args) < 2, "Too many parameters"); + for (size_type idx = start_idx; idx < end_idx; ++idx) { + auto element_address = &base_type::template internal_subscript(idx); + // try_call API is not convenient here due to broken + // variadic capture on GCC 4.8.5 + auto value_guard = make_raii_guard( [&] { + segment_index_type last_allocated_segment = this->find_last_allocated_segment(table); + size_type segment_size = this->segment_size(last_allocated_segment); + end_idx = end_idx < segment_size ? end_idx : segment_size; + for (size_type i = idx; i < end_idx; ++i) { + zero_unconstructed_elements(&this->internal_subscript(i), /*count =*/1); + } + }); + segment_table_allocator_traits::construct(base_type::get_allocator(), element_address, args...); + value_guard.dismiss(); + } + } + + template + void internal_loop_construct( segment_table_type table, size_type start_idx, size_type end_idx, ForwardIterator first, ForwardIterator ) { + for (size_type idx = start_idx; idx < end_idx; ++idx) { + auto element_address = &base_type::template internal_subscript(idx); + try_call( [&] { + segment_table_allocator_traits::construct(base_type::get_allocator(), element_address, *first++); + } ).on_exception( [&] { + segment_index_type last_allocated_segment = this->find_last_allocated_segment(table); + size_type segment_size = this->segment_size(last_allocated_segment); + end_idx = end_idx < segment_size ? end_idx : segment_size; + for (size_type i = idx; i < end_idx; ++i) { + zero_unconstructed_elements(&this->internal_subscript(i), /*count =*/1); + } + }); + } + } + + template + iterator internal_grow( size_type start_idx, size_type end_idx, const Args&... args ) { + size_type seg_index = this->segment_index_of(end_idx - 1); + this->assign_first_block_if_necessary(seg_index + 1); + segment_table_type table = this->get_table(); + this->extend_table_if_necessary(table, start_idx, end_idx); + + if (seg_index > this->my_first_block.load(std::memory_order_relaxed)) { + // So that other threads be able to work with the last segment of grow_by, allocate it immediately. + // If the last segment is not less than the first block + if (table[seg_index].load(std::memory_order_relaxed) == nullptr) { + size_type first_element = this->segment_base(seg_index); + if (first_element >= start_idx && first_element < end_idx) { + segment_type segment = table[seg_index].load(std::memory_order_relaxed); + base_type::enable_segment(segment, table, seg_index, first_element); + } + } + } + + internal_loop_construct(table, start_idx, end_idx, args...); + + return iterator(*this, start_idx, &base_type::template internal_subscript(start_idx)); + } + + + template + iterator internal_grow_by_delta( size_type delta, const Args&... args ) { + if (delta == size_type(0)) { + return end(); + } + size_type start_idx = this->my_size.fetch_add(delta); + size_type end_idx = start_idx + delta; + + return internal_grow(start_idx, end_idx, args...); + } + + template + iterator internal_grow_to_at_least( size_type new_size, const Args&... args ) { + size_type old_size = this->my_size.load(std::memory_order_relaxed); + if (new_size == size_type(0)) return iterator(*this, 0); + while (old_size < new_size && !this->my_size.compare_exchange_weak(old_size, new_size)) + {} + + int delta = static_cast(new_size) - static_cast(old_size); + if (delta > 0) { + return internal_grow(old_size, new_size, args...); + } + + size_type end_segment = this->segment_index_of(new_size - 1); + + // Check/wait for segments allocation completes + if (end_segment >= this->pointers_per_embedded_table && + this->get_table() == this->my_embedded_table) + { + spin_wait_while_eq(this->my_segment_table, this->my_embedded_table); + } + + for (segment_index_type seg_idx = 0; seg_idx <= end_segment; ++seg_idx) { + if (this->get_table()[seg_idx].load(std::memory_order_relaxed) == nullptr) { + atomic_backoff backoff(true); + while (this->get_table()[seg_idx].load(std::memory_order_relaxed) == nullptr) { + backoff.pause(); + } + } + } + + #if TBB_USE_DEBUG + size_type cap = capacity(); + __TBB_ASSERT( cap >= new_size, nullptr); + #endif + return iterator(*this, size()); + } + + template + void internal_resize( size_type n, const Args&... args ) { + if (n == 0) { + clear(); + return; + } + + size_type old_size = this->my_size.load(std::memory_order_acquire); + if (n > old_size) { + reserve(n); + grow_to_at_least(n, args...); + } else { + if (old_size == n) { + return; + } + size_type last_segment = this->segment_index_of(old_size - 1); + // Delete segments + for (size_type seg_idx = this->segment_index_of(n - 1) + 1; seg_idx <= last_segment; ++seg_idx) { + this->delete_segment(seg_idx); + } + + // If n > segment_size(n) => we need to destroy all of the items in the first segment + // Otherwise, we need to destroy only items with the index < n + size_type n_segment = this->segment_index_of(n - 1); + size_type last_index_to_destroy = std::min(this->segment_base(n_segment) + this->segment_size(n_segment), old_size); + // Destroy elements in curr segment + for (size_type idx = n; idx < last_index_to_destroy; ++idx) { + segment_table_allocator_traits::destroy(base_type::get_allocator(), &base_type::template internal_subscript(idx)); + } + this->my_size.store(n, std::memory_order_release); + } + } + + void destroy_elements() { + allocator_type alloc(base_type::get_allocator()); + for (size_type i = 0; i < this->my_size.load(std::memory_order_relaxed); ++i) { + allocator_traits_type::destroy(alloc, &base_type::template internal_subscript(i)); + } + this->my_size.store(0, std::memory_order_relaxed); + } + + static bool incompact_predicate( size_type size ) { + // memory page size + const size_type page_size = 4096; + return size < page_size || ((size - 1) % page_size < page_size / 2 && size < page_size * 128); + } + + void internal_compact() { + const size_type curr_size = this->my_size.load(std::memory_order_relaxed); + segment_table_type table = this->get_table(); + const segment_index_type k_end = this->find_last_allocated_segment(table); // allocated segments + const segment_index_type k_stop = curr_size ? this->segment_index_of(curr_size - 1) + 1 : 0; // number of segments to store existing items: 0=>0; 1,2=>1; 3,4=>2; [5-8]=>3;.. + const segment_index_type first_block = this->my_first_block; // number of merged segments, getting values from atomics + + segment_index_type k = first_block; + if (k_stop < first_block) { + k = k_stop; + } + else { + while (k < k_stop && incompact_predicate(this->segment_size(k) * sizeof(value_type))) k++; + } + + if (k_stop == k_end && k == first_block) { + return; + } + + // First segment optimization + if (k != first_block && k) { + size_type max_block = std::max(first_block, k); + + auto buffer_table = segment_table_allocator_traits::allocate(base_type::get_allocator(), max_block); + + for (size_type seg_idx = 0; seg_idx < max_block; ++seg_idx) { + segment_table_allocator_traits::construct(base_type::get_allocator(), &buffer_table[seg_idx], + table[seg_idx].load(std::memory_order_relaxed)); + table[seg_idx].store(nullptr, std::memory_order_relaxed); + } + + this->my_first_block.store(k, std::memory_order_relaxed); + size_type index = 0; + try_call( [&] { + for (; index < std::min(this->segment_size(max_block), curr_size); ++index) { + auto element_address = &static_cast(this)->operator[](index); + segment_index_type seg_idx = this->segment_index_of(index); + segment_table_allocator_traits::construct(base_type::get_allocator(), element_address, + std::move_if_noexcept(buffer_table[seg_idx].load(std::memory_order_relaxed)[index])); + } + } ).on_exception( [&] { + segment_element_allocator_type allocator(base_type::get_allocator()); + for (size_type i = 0; i < index; ++i) { + auto element_adress = &this->operator[](i); + segment_element_allocator_traits::destroy(allocator, element_adress); + } + segment_element_allocator_traits::deallocate(allocator, + table[0].load(std::memory_order_relaxed), this->segment_size(max_block)); + + for (size_type seg_idx = 0; seg_idx < max_block; ++seg_idx) { + table[seg_idx].store(buffer_table[seg_idx].load(std::memory_order_relaxed), + std::memory_order_relaxed); + buffer_table[seg_idx].store(nullptr, std::memory_order_relaxed); + } + segment_table_allocator_traits::deallocate(base_type::get_allocator(), + buffer_table, max_block); + this->my_first_block.store(first_block, std::memory_order_relaxed); + }); + + // Need to correct deallocate old segments + // Method destroy_segment respect active first_block, therefore, + // in order for the segment deletion to work correctly, set the first_block size that was earlier, + // destroy the unnecessary segments. + this->my_first_block.store(first_block, std::memory_order_relaxed); + for (size_type seg_idx = max_block; seg_idx > 0 ; --seg_idx) { + auto curr_segment = buffer_table[seg_idx - 1].load(std::memory_order_relaxed); + if (curr_segment != nullptr) { + destroy_segment(buffer_table[seg_idx - 1].load(std::memory_order_relaxed) + this->segment_base(seg_idx - 1), + seg_idx - 1); + } + } + + this->my_first_block.store(k, std::memory_order_relaxed); + + for (size_type seg_idx = 0; seg_idx < max_block; ++seg_idx) { + segment_table_allocator_traits::destroy(base_type::get_allocator(), &buffer_table[seg_idx]); + } + + segment_table_allocator_traits::deallocate(base_type::get_allocator(), buffer_table, max_block); + } + // free unnecessary segments allocated by reserve() call + if (k_stop < k_end) { + for (size_type seg_idx = k_end; seg_idx != k_stop; --seg_idx) { + if (table[seg_idx - 1].load(std::memory_order_relaxed) != nullptr) { + this->delete_segment(seg_idx - 1); + } + } + if (!k) this->my_first_block.store(0, std::memory_order_relaxed); + } + } + + // Lever for adjusting the size of first_block at the very first insertion. + // TODO: consider >1 value, check performance + static constexpr size_type default_first_block_size = 1; + + template + friend class vector_iterator; +}; // class concurrent_vector + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +// Deduction guide for the constructor from two iterators +template >, + typename = std::enable_if_t>, + typename = std::enable_if_t>> +concurrent_vector( It, It, Alloc = Alloc() ) +-> concurrent_vector, Alloc>; +#endif + +template +void swap(concurrent_vector &lhs, + concurrent_vector &rhs) +{ + lhs.swap(rhs); +} + +template +bool operator==(const concurrent_vector &lhs, + const concurrent_vector &rhs) +{ + return lhs.size() == rhs.size() && std::equal(lhs.begin(), lhs.end(), rhs.begin()); +} + +#if !__TBB_CPP20_COMPARISONS_PRESENT +template +bool operator!=(const concurrent_vector &lhs, + const concurrent_vector &rhs) +{ + return !(lhs == rhs); +} +#endif // !__TBB_CPP20_COMPARISONS_PRESENT + +#if __TBB_CPP20_COMPARISONS_PRESENT && __TBB_CPP20_CONCEPTS_PRESENT +template +tbb::detail::synthesized_three_way_result::value_type> +operator<=>(const concurrent_vector &lhs, + const concurrent_vector &rhs) +{ + return std::lexicographical_compare_three_way(lhs.begin(), lhs.end(), + rhs.begin(), rhs.end(), + tbb::detail::synthesized_three_way_comparator{}); +} + +#else + +template +bool operator<(const concurrent_vector &lhs, + const concurrent_vector &rhs) +{ + return std::lexicographical_compare(lhs.begin(), lhs.end(), rhs.begin(), rhs.end()); +} + +template +bool operator<=(const concurrent_vector &lhs, + const concurrent_vector &rhs) +{ + return !(rhs < lhs); +} + +template +bool operator>(const concurrent_vector &lhs, + const concurrent_vector &rhs) +{ + return rhs < lhs; +} + +template +bool operator>=(const concurrent_vector &lhs, + const concurrent_vector &rhs) +{ + return !(lhs < rhs); +} +#endif // __TBB_CPP20_COMPARISONS_PRESENT && __TBB_CPP20_CONCEPTS_PRESENT + +} // namespace d1 +} // namespace detail + +inline namespace v1 { + using detail::d1::concurrent_vector; +} // namespace v1 + +} // namespace tbb + +#endif // __TBB_concurrent_vector_H diff --git a/src/3rd party/oneapi/tbb/detail/_aggregator.h b/src/3rd party/oneapi/tbb/detail/_aggregator.h new file mode 100644 index 0000000000..2e3e5cc3d6 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_aggregator.h @@ -0,0 +1,176 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + + +#ifndef __TBB_detail__aggregator_H +#define __TBB_detail__aggregator_H + +#include "_assert.h" +#include "_utils.h" +#include +#if !__TBBMALLOC_BUILD // TODO: check this macro with TBB Malloc +#include "../profiling.h" +#endif + +namespace tbb { +namespace detail { +namespace d1 { + +// Base class for aggregated operation +template +class aggregated_operation { +public: + // Zero value means "wait" status, all other values are "user" specified values and + // are defined into the scope of a class which uses "status" + std::atomic status; + + std::atomic next; + aggregated_operation() : status{}, next(nullptr) {} +}; // class aggregated_operation + +// Aggregator base class +/* An aggregator for collecting operations coming from multiple sources and executing + them serially on a single thread. OperationType must be derived from + aggregated_operation. The parameter HandlerType is a functor that will be passed the + list of operations and is expected to handle each operation appropriately, setting the + status of each operation to non-zero. */ +template +class aggregator_generic { +public: + aggregator_generic() : pending_operations(nullptr), handler_busy(false) {} + + // Execute an operation + /* Places an operation into the waitlist (pending_operations), and either handles the list, + or waits for the operation to complete, or returns. + The long_life_time parameter specifies the life time of the given operation object. + Operations with long_life_time == true may be accessed after execution. + A "short" life time operation (long_life_time == false) can be destroyed + during execution, and so any access to it after it was put into the waitlist, + including status check, is invalid. As a consequence, waiting for completion + of such operation causes undefined behavior. */ + template + void execute( OperationType* op, HandlerType& handle_operations, bool long_life_time = true ) { + // op->status should be read before inserting the operation into the + // aggregator waitlist since it can become invalid after executing a + // handler (if the operation has 'short' life time.) + const uintptr_t status = op->status.load(std::memory_order_relaxed); + + // ITT note: &(op->status) tag is used to cover accesses to this op node. This + // thread has created the operation, and now releases it so that the handler + // thread may handle the associated operation w/o triggering a race condition; + // thus this tag will be acquired just before the operation is handled in the + // handle_operations functor. + call_itt_notify(releasing, &(op->status)); + // insert the operation in the queue. + OperationType* res = pending_operations.load(std::memory_order_relaxed); + do { + op->next.store(res, std::memory_order_relaxed); + } while (!pending_operations.compare_exchange_strong(res, op)); + if (!res) { // first in the list; handle the operations + // ITT note: &pending_operations tag covers access to the handler_busy flag, + // which this waiting handler thread will try to set before entering + // handle_operations. + call_itt_notify(acquired, &pending_operations); + start_handle_operations(handle_operations); + // The operation with 'short' life time can already be destroyed + if (long_life_time) + __TBB_ASSERT(op->status.load(std::memory_order_relaxed), nullptr); + } + // Not first; wait for op to be ready + else if (!status) { // operation is blocking here. + __TBB_ASSERT(long_life_time, "Waiting for an operation object that might be destroyed during processing"); + call_itt_notify(prepare, &(op->status)); + spin_wait_while_eq(op->status, uintptr_t(0)); + } + } + +private: + // Trigger the handling of operations when the handler is free + template + void start_handle_operations( HandlerType& handle_operations ) { + OperationType* op_list; + + // ITT note: &handler_busy tag covers access to pending_operations as it is passed + // between active and waiting handlers. Below, the waiting handler waits until + // the active handler releases, and the waiting handler acquires &handler_busy as + // it becomes the active_handler. The release point is at the end of this + // function, when all operations in pending_operations have been handled by the + // owner of this aggregator. + call_itt_notify(prepare, &handler_busy); + // get the handler_busy: + // only one thread can possibly spin here at a time + spin_wait_until_eq(handler_busy, uintptr_t(0)); + call_itt_notify(acquired, &handler_busy); + // acquire fence not necessary here due to causality rule and surrounding atomics + handler_busy.store(1, std::memory_order_relaxed); + + // ITT note: &pending_operations tag covers access to the handler_busy flag + // itself. Capturing the state of the pending_operations signifies that + // handler_busy has been set and a new active handler will now process that list's + // operations. + call_itt_notify(releasing, &pending_operations); + // grab pending_operations + op_list = pending_operations.exchange(nullptr); + + // handle all the operations + handle_operations(op_list); + + // release the handler + handler_busy.store(0, std::memory_order_release); + } + + // An atomically updated list (aka mailbox) of pending operations + std::atomic pending_operations; + // Controls threads access to handle_operations + std::atomic handler_busy; +}; // class aggregator_generic + +template +class aggregator : public aggregator_generic { + HandlerType handle_operations; +public: + aggregator() = default; + + void initialize_handler( HandlerType h ) { handle_operations = h; } + + void execute(OperationType* op) { + aggregator_generic::execute(op, handle_operations); + } +}; // class aggregator + +// the most-compatible friend declaration (vs, gcc, icc) is +// template friend class aggregating_functor; +template +class aggregating_functor { + AggregatingClass* my_object{nullptr}; +public: + aggregating_functor() = default; + aggregating_functor( AggregatingClass* object ) : my_object(object) { + __TBB_ASSERT(my_object, nullptr); + } + + void operator()( OperationList* op_list ) { + __TBB_ASSERT(my_object, nullptr); + my_object->handle_operations(op_list); + } +}; // class aggregating_functor + + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__aggregator_H diff --git a/src/3rd party/oneapi/tbb/detail/_aligned_space.h b/src/3rd party/oneapi/tbb/detail/_aligned_space.h new file mode 100644 index 0000000000..13857c47cc --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_aligned_space.h @@ -0,0 +1,46 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#ifndef __TBB_aligned_space_H +#define __TBB_aligned_space_H + +#include + +#include "_template_helpers.h" + +namespace tbb { +namespace detail { +inline namespace d0 { + +//! Block of space aligned sufficiently to construct an array T with N elements. +/** The elements are not constructed or destroyed by this class. + @ingroup memory_allocation */ +template +class aligned_space { + alignas(alignof(T)) std::uint8_t aligned_array[N * sizeof(T)]; + +public: + //! Pointer to beginning of array + T* begin() const { return punned_cast(&aligned_array); } + + //! Pointer to one past last element in array. + T* end() const { return begin() + N; } +}; + +} // namespace d0 +} // namespace detail +} // namespace tbb + +#endif /* __TBB_aligned_space_H */ diff --git a/src/3rd party/oneapi/tbb/detail/_allocator_traits.h b/src/3rd party/oneapi/tbb/detail/_allocator_traits.h new file mode 100644 index 0000000000..8c60e25e7e --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_allocator_traits.h @@ -0,0 +1,107 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__allocator_traits_H +#define __TBB_detail__allocator_traits_H + +#include "_config.h" +#include "_template_helpers.h" +#include +#include + +namespace tbb { +namespace detail { +inline namespace d0 { + +#if !__TBB_CPP17_ALLOCATOR_IS_ALWAYS_EQUAL_PRESENT +// Struct is_always_equal_detector provides the member type "type" which is +// Allocator::is_always_equal if it is present, std::false_type otherwise +template +struct is_always_equal_detector { + using type = std::false_type; +}; + +template +struct is_always_equal_detector> +{ + using type = typename Allocator::is_always_equal; +}; +#endif // !__TBB_CPP17_ALLOCATOR_IS_ALWAYS_EQUAL_PRESENT + +template +class allocator_traits : public std::allocator_traits +{ + using base_type = std::allocator_traits; +public: +#if !__TBB_CPP17_ALLOCATOR_IS_ALWAYS_EQUAL_PRESENT + using is_always_equal = typename is_always_equal_detector::type; +#endif + + template + using rebind_traits = typename tbb::detail::allocator_traits>; +}; // struct allocator_traits + +template +void copy_assign_allocators_impl( Allocator& lhs, const Allocator& rhs, /*pocca = */std::true_type ) { + lhs = rhs; +} + +template +void copy_assign_allocators_impl( Allocator&, const Allocator&, /*pocca = */ std::false_type ) {} + +// Copy assigns allocators only if propagate_on_container_copy_assignment is true +template +void copy_assign_allocators( Allocator& lhs, const Allocator& rhs ) { + using pocca_type = typename allocator_traits::propagate_on_container_copy_assignment; + copy_assign_allocators_impl(lhs, rhs, pocca_type()); +} + +template +void move_assign_allocators_impl( Allocator& lhs, Allocator& rhs, /*pocma = */ std::true_type ) { + lhs = std::move(rhs); +} + +template +void move_assign_allocators_impl( Allocator&, Allocator&, /*pocma = */ std::false_type ) {} + +// Move assigns allocators only if propagate_on_container_move_assignment is true +template +void move_assign_allocators( Allocator& lhs, Allocator& rhs ) { + using pocma_type = typename allocator_traits::propagate_on_container_move_assignment; + move_assign_allocators_impl(lhs, rhs, pocma_type()); +} + +template +void swap_allocators_impl( Allocator& lhs, Allocator& rhs, /*pocs = */ std::true_type ) { + using std::swap; + swap(lhs, rhs); +} + +template +void swap_allocators_impl( Allocator&, Allocator&, /*pocs = */ std::false_type ) {} + +// Swaps allocators only if propagate_on_container_swap is true +template +void swap_allocators( Allocator& lhs, Allocator& rhs ) { + using pocs_type = typename allocator_traits::propagate_on_container_swap; + swap_allocators_impl(lhs, rhs, pocs_type()); +} + +} // inline namespace d0 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__allocator_traits_H diff --git a/src/3rd party/oneapi/tbb/detail/_assert.h b/src/3rd party/oneapi/tbb/detail/_assert.h new file mode 100644 index 0000000000..494229fc46 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_assert.h @@ -0,0 +1,70 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + Copyright (c) 2025 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__assert_H +#define __TBB_detail__assert_H + +#include "_config.h" + +#if __TBBMALLOC_BUILD +namespace rml { namespace internal { +#else +namespace tbb { +namespace detail { +namespace r1 { +#endif +//! Process an assertion failure. +/** Normally called from __TBB_ASSERT macro. + If assertion handler is not set, the default one is used, which prints the message for assertion + failure and aborts. The setting of an assertion handler is available as part of oneTBB + specification extension. */ +TBB_EXPORT void __TBB_EXPORTED_FUNC assertion_failure(const char *location, int line, + const char *expression, const char *comment); + +using assertion_handler_type = decltype(&assertion_failure); + +#if __TBBMALLOC_BUILD +}} // namespaces rml::internal +#else +} // namespace r1 +} // namespace detail +} // namespace tbb +#endif + +#if __TBBMALLOC_BUILD +//! Release version of assertions +#define __TBB_ASSERT_RELEASE(predicate,message) ((predicate)?((void)0) : rml::internal::assertion_failure(__func__,__LINE__,#predicate,message)) +#else +#define __TBB_ASSERT_RELEASE(predicate,message) ((predicate)?((void)0) : tbb::detail::r1::assertion_failure(__func__,__LINE__,#predicate,message)) +#endif + +#if TBB_USE_ASSERT + //! Assert that predicate is true. + /** If predicate is false, print assertion failure message. + If the comment argument is not nullptr, it is printed as part of the failure message. + The comment argument has no other effect. */ + #define __TBB_ASSERT(predicate,message) __TBB_ASSERT_RELEASE(predicate,message) + //! "Extended" version + #define __TBB_ASSERT_EX __TBB_ASSERT +#else + //! No-op version of __TBB_ASSERT. + #define __TBB_ASSERT(predicate,comment) ((void)0) + //! "Extended" version is useful to suppress warnings if a variable is only used with an assert + #define __TBB_ASSERT_EX(predicate,comment) ((void)(1 && (predicate))) +#endif // TBB_USE_ASSERT + +#endif // __TBB_detail__assert_H diff --git a/src/3rd party/tbb/internal/_warning_suppress_disable_notice.h b/src/3rd party/oneapi/tbb/detail/_attach.h similarity index 65% rename from src/3rd party/tbb/internal/_warning_suppress_disable_notice.h rename to src/3rd party/oneapi/tbb/detail/_attach.h index fed04a15a3..45f29727a0 100644 --- a/src/3rd party/tbb/internal/_warning_suppress_disable_notice.h +++ b/src/3rd party/oneapi/tbb/detail/_attach.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,14 +14,19 @@ limitations under the License. */ -#if __TBB_SUPPRESS_INTERNAL_DEPRECATED_MESSAGES +#ifndef __TBB_detail__attach_H +#define __TBB_detail__attach_H -#if __INTEL_COMPILER || _MSC_VER -#pragma warning( pop ) -#elif __GNUC__ -#pragma GCC diagnostic pop -#elif __clang__ -#pragma clang diagnostic pop -#endif +#include "_config.h" -#endif // __TBB_SUPPRESS_INTERNAL_DEPRECATED_MESSAGES +namespace tbb { +namespace detail { +namespace d1 { + + struct attach {}; + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__attach_H diff --git a/src/3rd party/oneapi/tbb/detail/_concurrent_queue_base.h b/src/3rd party/oneapi/tbb/detail/_concurrent_queue_base.h new file mode 100644 index 0000000000..ee628e1e89 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_concurrent_queue_base.h @@ -0,0 +1,650 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__concurrent_queue_base_H +#define __TBB_detail__concurrent_queue_base_H + +#include "_utils.h" +#include "_exception.h" +#include "_machine.h" +#include "_allocator_traits.h" + +#include "../profiling.h" +#include "../spin_mutex.h" +#include "../cache_aligned_allocator.h" + +#include + +namespace tbb { +namespace detail { +namespace d2 { + +using ticket_type = std::size_t; + +template +inline bool is_valid_page(const Page p) { + return reinterpret_cast(p) > 1; +} + +template +struct concurrent_queue_rep; + +template +class micro_queue_pop_finalizer; + +#if _MSC_VER && !defined(__INTEL_COMPILER) +// unary minus operator applied to unsigned type, result still unsigned +#pragma warning( push ) +#pragma warning( disable: 4146 ) +#endif + +// A queue using simple locking. +// For efficiency, this class has no constructor. +// The caller is expected to zero-initialize it. +template +class micro_queue { +private: + using queue_rep_type = concurrent_queue_rep; + using self_type = micro_queue; +public: + using size_type = std::size_t; + using value_type = T; + using reference = value_type&; + using const_reference = const value_type&; + + using allocator_type = Allocator; + using allocator_traits_type = tbb::detail::allocator_traits; + using queue_allocator_type = typename allocator_traits_type::template rebind_alloc; + + static constexpr size_type item_size = sizeof(T); + static constexpr size_type items_per_page = item_size <= 8 ? 32 : + item_size <= 16 ? 16 : + item_size <= 32 ? 8 : + item_size <= 64 ? 4 : + item_size <= 128 ? 2 : 1; + + struct padded_page { + padded_page() {} + ~padded_page() {} + + reference operator[] (std::size_t index) { + __TBB_ASSERT(index < items_per_page, "Index out of range"); + return items[index]; + } + + const_reference operator[] (std::size_t index) const { + __TBB_ASSERT(index < items_per_page, "Index out of range"); + return items[index]; + } + + padded_page* next{ nullptr }; + std::atomic mask{}; + + union { + value_type items[items_per_page]; + }; + }; // struct padded_page + + using page_allocator_type = typename allocator_traits_type::template rebind_alloc; +protected: + using page_allocator_traits = tbb::detail::allocator_traits; + +public: + using item_constructor_type = void (*)(value_type* location, const void* src); + micro_queue() = default; + micro_queue( const micro_queue& ) = delete; + micro_queue& operator=( const micro_queue& ) = delete; + + size_type prepare_page( ticket_type k, queue_rep_type& base, page_allocator_type page_allocator, + padded_page*& p ) { + __TBB_ASSERT(p == nullptr, "Invalid page argument for prepare_page"); + k &= -queue_rep_type::n_queue; + size_type index = modulo_power_of_two(k / queue_rep_type::n_queue, items_per_page); + if (!index) { + try_call( [&] { + p = page_allocator_traits::allocate(page_allocator, 1); + }).on_exception( [&] { + ++base.n_invalid_entries; + invalidate_page( k ); + }); + page_allocator_traits::construct(page_allocator, p); + } + + spin_wait_until_my_turn(tail_counter, k, base); + d1::call_itt_notify(d1::acquired, &tail_counter); + + if (p) { + spin_mutex::scoped_lock lock( page_mutex ); + padded_page* q = tail_page.load(std::memory_order_relaxed); + if (is_valid_page(q)) { + q->next = p; + } else { + head_page.store(p, std::memory_order_relaxed); + } + tail_page.store(p, std::memory_order_relaxed); + } else { + p = tail_page.load(std::memory_order_relaxed); + } + return index; + } + + template + void push( ticket_type k, queue_rep_type& base, queue_allocator_type& allocator, Args&&... args ) + { + padded_page* p = nullptr; + page_allocator_type page_allocator(allocator); + size_type index = prepare_page(k, base, page_allocator, p); + __TBB_ASSERT(p != nullptr, "Page was not prepared"); + + // try_call API is not convenient here due to broken + // variadic capture on GCC 4.8.5 + auto value_guard = make_raii_guard([&] { + ++base.n_invalid_entries; + d1::call_itt_notify(d1::releasing, &tail_counter); + tail_counter.fetch_add(queue_rep_type::n_queue); + }); + + page_allocator_traits::construct(page_allocator, &(*p)[index], std::forward(args)...); + // If no exception was thrown, mark item as present. + p->mask.store(p->mask.load(std::memory_order_relaxed) | uintptr_t(1) << index, std::memory_order_relaxed); + d1::call_itt_notify(d1::releasing, &tail_counter); + + value_guard.dismiss(); + tail_counter.fetch_add(queue_rep_type::n_queue); + } + + void abort_push( ticket_type k, queue_rep_type& base, queue_allocator_type& allocator ) { + padded_page* p = nullptr; + prepare_page(k, base, allocator, p); + ++base.n_invalid_entries; + tail_counter.fetch_add(queue_rep_type::n_queue); + } + + bool pop( void* dst, ticket_type k, queue_rep_type& base, queue_allocator_type& allocator ) { + k &= -queue_rep_type::n_queue; + spin_wait_until_eq(head_counter, k); + d1::call_itt_notify(d1::acquired, &head_counter); + spin_wait_while_eq(tail_counter, k); + d1::call_itt_notify(d1::acquired, &tail_counter); + padded_page *p = head_page.load(std::memory_order_relaxed); + __TBB_ASSERT( p, nullptr ); + size_type index = modulo_power_of_two( k/queue_rep_type::n_queue, items_per_page ); + bool success = false; + { + page_allocator_type page_allocator(allocator); + micro_queue_pop_finalizer finalizer(*this, page_allocator, + k + queue_rep_type::n_queue, index == items_per_page - 1 ? p : nullptr ); + if (p->mask.load(std::memory_order_relaxed) & (std::uintptr_t(1) << index)) { + success = true; + assign_and_destroy_item(dst, *p, index); + } else { + --base.n_invalid_entries; + } + } + return success; + } + + micro_queue& assign( const micro_queue& src, queue_allocator_type& allocator, + item_constructor_type construct_item ) + { + head_counter.store(src.head_counter.load(std::memory_order_relaxed), std::memory_order_relaxed); + tail_counter.store(src.tail_counter.load(std::memory_order_relaxed), std::memory_order_relaxed); + + const padded_page* srcp = src.head_page.load(std::memory_order_relaxed); + if( is_valid_page(srcp) ) { + ticket_type g_index = head_counter.load(std::memory_order_relaxed); + size_type n_items = (tail_counter.load(std::memory_order_relaxed) - head_counter.load(std::memory_order_relaxed)) + / queue_rep_type::n_queue; + size_type index = modulo_power_of_two(head_counter.load(std::memory_order_relaxed) / queue_rep_type::n_queue, items_per_page); + size_type end_in_first_page = (index+n_items < items_per_page) ? (index + n_items) : items_per_page; + + try_call( [&] { + head_page.store(make_copy(allocator, srcp, index, end_in_first_page, g_index, construct_item), std::memory_order_relaxed); + }).on_exception( [&] { + head_counter.store(0, std::memory_order_relaxed); + tail_counter.store(0, std::memory_order_relaxed); + }); + padded_page* cur_page = head_page.load(std::memory_order_relaxed); + + try_call( [&] { + if (srcp != src.tail_page.load(std::memory_order_relaxed)) { + for (srcp = srcp->next; srcp != src.tail_page.load(std::memory_order_relaxed); srcp=srcp->next ) { + cur_page->next = make_copy( allocator, srcp, 0, items_per_page, g_index, construct_item ); + cur_page = cur_page->next; + } + + __TBB_ASSERT(srcp == src.tail_page.load(std::memory_order_relaxed), nullptr ); + size_type last_index = modulo_power_of_two(tail_counter.load(std::memory_order_relaxed) / queue_rep_type::n_queue, items_per_page); + if( last_index==0 ) last_index = items_per_page; + + cur_page->next = make_copy( allocator, srcp, 0, last_index, g_index, construct_item ); + cur_page = cur_page->next; + } + tail_page.store(cur_page, std::memory_order_relaxed); + }).on_exception( [&] { + padded_page* invalid_page = reinterpret_cast(std::uintptr_t(1)); + tail_page.store(invalid_page, std::memory_order_relaxed); + }); + } else { + head_page.store(nullptr, std::memory_order_relaxed); + tail_page.store(nullptr, std::memory_order_relaxed); + } + return *this; + } + + padded_page* make_copy( queue_allocator_type& allocator, const padded_page* src_page, size_type begin_in_page, + size_type end_in_page, ticket_type& g_index, item_constructor_type construct_item ) + { + page_allocator_type page_allocator(allocator); + padded_page* new_page = page_allocator_traits::allocate(page_allocator, 1); + new_page->next = nullptr; + new_page->mask.store(src_page->mask.load(std::memory_order_relaxed), std::memory_order_relaxed); + for (; begin_in_page!=end_in_page; ++begin_in_page, ++g_index) { + if (new_page->mask.load(std::memory_order_relaxed) & uintptr_t(1) << begin_in_page) { + copy_item(*new_page, begin_in_page, *src_page, begin_in_page, construct_item); + } + } + return new_page; + } + + void invalidate_page( ticket_type k ) { + // Append an invalid page at address 1 so that no more pushes are allowed. + padded_page* invalid_page = reinterpret_cast(std::uintptr_t(1)); + { + spin_mutex::scoped_lock lock( page_mutex ); + tail_counter.store(k + queue_rep_type::n_queue + 1, std::memory_order_relaxed); + padded_page* q = tail_page.load(std::memory_order_relaxed); + if (is_valid_page(q)) { + q->next = invalid_page; + } else { + head_page.store(invalid_page, std::memory_order_relaxed); + } + tail_page.store(invalid_page, std::memory_order_relaxed); + } + } + + padded_page* get_head_page() { + return head_page.load(std::memory_order_relaxed); + } + + void clear(queue_allocator_type& allocator, padded_page* new_head = nullptr, padded_page* new_tail = nullptr) { + padded_page* curr_page = get_head_page(); + size_type index = (head_counter.load(std::memory_order_relaxed) / queue_rep_type::n_queue) % items_per_page; + page_allocator_type page_allocator(allocator); + + while (curr_page && is_valid_page(curr_page)) { + while (index != items_per_page) { + if (curr_page->mask.load(std::memory_order_relaxed) & (std::uintptr_t(1) << index)) { + page_allocator_traits::destroy(page_allocator, &curr_page->operator[](index)); + } + ++index; + } + + index = 0; + padded_page* next_page = curr_page->next; + page_allocator_traits::destroy(page_allocator, curr_page); + page_allocator_traits::deallocate(page_allocator, curr_page, 1); + curr_page = next_page; + } + head_counter.store(0, std::memory_order_relaxed); + tail_counter.store(0, std::memory_order_relaxed); + head_page.store(new_head, std::memory_order_relaxed); + tail_page.store(new_tail, std::memory_order_relaxed); + } + + void clear_and_invalidate(queue_allocator_type& allocator) { + padded_page* invalid_page = reinterpret_cast(std::uintptr_t(1)); + clear(allocator, invalid_page, invalid_page); + } + +private: + // template + friend class micro_queue_pop_finalizer; + + // Class used to ensure exception-safety of method "pop" + class destroyer { + value_type& my_value; + public: + destroyer( reference value ) : my_value(value) {} + destroyer( const destroyer& ) = delete; + destroyer& operator=( const destroyer& ) = delete; + ~destroyer() {my_value.~T();} + }; // class destroyer + + void copy_item( padded_page& dst, size_type dindex, const padded_page& src, size_type sindex, + item_constructor_type construct_item ) + { + auto& src_item = src[sindex]; + construct_item( &dst[dindex], static_cast(&src_item) ); + } + + void assign_and_destroy_item( void* dst, padded_page& src, size_type index ) { + auto& from = src[index]; + destroyer d(from); + *static_cast(dst) = std::move(from); + } + + void spin_wait_until_my_turn( std::atomic& counter, ticket_type k, queue_rep_type& rb ) const { + for (atomic_backoff b{};; b.pause()) { + ticket_type c = counter.load(std::memory_order_acquire); + if (c == k) return; + else if (c & 1) { + ++rb.n_invalid_entries; + throw_exception( exception_id::bad_last_alloc); + } + } + } + + std::atomic head_page{}; + std::atomic head_counter{}; + + std::atomic tail_page{}; + std::atomic tail_counter{}; + + spin_mutex page_mutex{}; +}; // class micro_queue + +#if _MSC_VER && !defined(__INTEL_COMPILER) +#pragma warning( pop ) +#endif // warning 4146 is back + +template +class micro_queue_pop_finalizer { +public: + using padded_page = typename Container::padded_page; + using allocator_type = Allocator; + using allocator_traits_type = tbb::detail::allocator_traits; + + micro_queue_pop_finalizer( Container& queue, Allocator& alloc, ticket_type k, padded_page* p ) : + my_ticket_type(k), my_queue(queue), my_page(p), allocator(alloc) + {} + + micro_queue_pop_finalizer( const micro_queue_pop_finalizer& ) = delete; + micro_queue_pop_finalizer& operator=( const micro_queue_pop_finalizer& ) = delete; + + ~micro_queue_pop_finalizer() { + padded_page* p = my_page; + if( is_valid_page(p) ) { + spin_mutex::scoped_lock lock( my_queue.page_mutex ); + padded_page* q = p->next; + my_queue.head_page.store(q, std::memory_order_relaxed); + if( !is_valid_page(q) ) { + my_queue.tail_page.store(nullptr, std::memory_order_relaxed); + } + } + my_queue.head_counter.store(my_ticket_type, std::memory_order_release); + if ( is_valid_page(p) ) { + allocator_traits_type::destroy(allocator, static_cast(p)); + allocator_traits_type::deallocate(allocator, static_cast(p), 1); + } + } +private: + ticket_type my_ticket_type; + Container& my_queue; + padded_page* my_page; + Allocator& allocator; +}; // class micro_queue_pop_finalizer + +#if _MSC_VER && !defined(__INTEL_COMPILER) +// structure was padded due to alignment specifier +#pragma warning( push ) +#pragma warning( disable: 4324 ) +#endif + +template +struct concurrent_queue_rep { + using self_type = concurrent_queue_rep; + using size_type = std::size_t; + using micro_queue_type = micro_queue; + using allocator_type = Allocator; + using allocator_traits_type = tbb::detail::allocator_traits; + using padded_page = typename micro_queue_type::padded_page; + using page_allocator_type = typename micro_queue_type::page_allocator_type; + using item_constructor_type = typename micro_queue_type::item_constructor_type; +private: + using page_allocator_traits = tbb::detail::allocator_traits; + using queue_allocator_type = typename allocator_traits_type::template rebind_alloc; + +public: + // must be power of 2 + static constexpr size_type n_queue = 8; + // Approximately n_queue/golden ratio + static constexpr size_type phi = 3; + static constexpr size_type item_size = micro_queue_type::item_size; + static constexpr size_type items_per_page = micro_queue_type::items_per_page; + + concurrent_queue_rep() {} + + concurrent_queue_rep( const concurrent_queue_rep& ) = delete; + concurrent_queue_rep& operator=( const concurrent_queue_rep& ) = delete; + + void clear( queue_allocator_type& alloc ) { + for (size_type index = 0; index < n_queue; ++index) { + array[index].clear(alloc); + } + head_counter.store(0, std::memory_order_relaxed); + tail_counter.store(0, std::memory_order_relaxed); + n_invalid_entries.store(0, std::memory_order_relaxed); + } + + void assign( const concurrent_queue_rep& src, queue_allocator_type& alloc, item_constructor_type construct_item ) { + head_counter.store(src.head_counter.load(std::memory_order_relaxed), std::memory_order_relaxed); + tail_counter.store(src.tail_counter.load(std::memory_order_relaxed), std::memory_order_relaxed); + n_invalid_entries.store(src.n_invalid_entries.load(std::memory_order_relaxed), std::memory_order_relaxed); + + // copy or move micro_queues + size_type queue_idx = 0; + try_call( [&] { + for (; queue_idx < n_queue; ++queue_idx) { + array[queue_idx].assign(src.array[queue_idx], alloc, construct_item); + } + }).on_exception( [&] { + for (size_type i = 0; i < queue_idx + 1; ++i) { + array[i].clear_and_invalidate(alloc); + } + head_counter.store(0, std::memory_order_relaxed); + tail_counter.store(0, std::memory_order_relaxed); + n_invalid_entries.store(0, std::memory_order_relaxed); + }); + + __TBB_ASSERT(head_counter.load(std::memory_order_relaxed) == src.head_counter.load(std::memory_order_relaxed) && + tail_counter.load(std::memory_order_relaxed) == src.tail_counter.load(std::memory_order_relaxed), + "the source concurrent queue should not be concurrently modified." ); + } + + bool empty() const { + ticket_type tc = tail_counter.load(std::memory_order_acquire); + ticket_type hc = head_counter.load(std::memory_order_relaxed); + // if tc!=r.tail_counter, the queue was not empty at some point between the two reads. + return tc == tail_counter.load(std::memory_order_relaxed) && + std::ptrdiff_t(tc - hc - n_invalid_entries.load(std::memory_order_relaxed)) <= 0; + } + + std::ptrdiff_t size() const { + __TBB_ASSERT(sizeof(std::ptrdiff_t) <= sizeof(size_type), nullptr); + std::ptrdiff_t hc = head_counter.load(std::memory_order_acquire); + std::ptrdiff_t tc = tail_counter.load(std::memory_order_relaxed); + std::ptrdiff_t nie = n_invalid_entries.load(std::memory_order_relaxed); + + return tc - hc - nie; + } + + friend class micro_queue; + + // Map ticket_type to an array index + static size_type index( ticket_type k ) { + return k * phi % n_queue; + } + + micro_queue_type& choose( ticket_type k ) { + // The formula here approximates LRU in a cache-oblivious way. + return array[index(k)]; + } + + alignas(max_nfs_size) micro_queue_type array[n_queue]; + + alignas(max_nfs_size) std::atomic head_counter{}; + alignas(max_nfs_size) std::atomic tail_counter{}; + alignas(max_nfs_size) std::atomic n_invalid_entries{}; +}; // class concurrent_queue_rep + +#if _MSC_VER && !defined(__INTEL_COMPILER) +#pragma warning( pop ) +#endif + +template +class concurrent_queue_iterator_base { + using queue_rep_type = concurrent_queue_rep; + using padded_page = typename queue_rep_type::padded_page; +protected: + concurrent_queue_iterator_base() = default; + + concurrent_queue_iterator_base( const concurrent_queue_iterator_base& other ) { + assign(other); + } + + concurrent_queue_iterator_base( queue_rep_type* queue_rep ) + : my_queue_rep(queue_rep), + my_head_counter(my_queue_rep->head_counter.load(std::memory_order_relaxed)) + { + for (std::size_t i = 0; i < queue_rep_type::n_queue; ++i) { + my_array[i] = my_queue_rep->array[i].get_head_page(); + } + + if (!get_item(my_item, my_head_counter)) advance(); + } + + void assign( const concurrent_queue_iterator_base& other ) { + my_item = other.my_item; + my_queue_rep = other.my_queue_rep; + + if (my_queue_rep != nullptr) { + my_head_counter = other.my_head_counter; + + for (std::size_t i = 0; i < queue_rep_type::n_queue; ++i) { + my_array[i] = other.my_array[i]; + } + } + } + + void advance() { + __TBB_ASSERT(my_item, "Attempt to increment iterator past end of the queue"); + std::size_t k = my_head_counter; +#if TBB_USE_ASSERT + Value* tmp; + get_item(tmp, k); + __TBB_ASSERT(my_item == tmp, nullptr); +#endif + std::size_t i = modulo_power_of_two(k / queue_rep_type::n_queue, my_queue_rep->items_per_page); + if (i == my_queue_rep->items_per_page - 1) { + padded_page*& root = my_array[queue_rep_type::index(k)]; + root = root->next; + } + // Advance k + my_head_counter = ++k; + if (!get_item(my_item, k)) advance(); + } + + concurrent_queue_iterator_base& operator=( const concurrent_queue_iterator_base& other ) { + this->assign(other); + return *this; + } + + bool get_item( Value*& item, std::size_t k ) { + if (k == my_queue_rep->tail_counter.load(std::memory_order_relaxed)) { + item = nullptr; + return true; + } else { + padded_page* p = my_array[queue_rep_type::index(k)]; + __TBB_ASSERT(p, nullptr); + std::size_t i = modulo_power_of_two(k / queue_rep_type::n_queue, my_queue_rep->items_per_page); + item = &(*p)[i]; + return (p->mask & uintptr_t(1) << i) != 0; + } + } + + Value* my_item{ nullptr }; + queue_rep_type* my_queue_rep{ nullptr }; + ticket_type my_head_counter{}; + padded_page* my_array[queue_rep_type::n_queue]{}; +}; // class concurrent_queue_iterator_base + +struct concurrent_queue_iterator_provider { + template + static Iterator get( const Container& container ) { + return Iterator(container); + } +}; // struct concurrent_queue_iterator_provider + +template +class concurrent_queue_iterator : public concurrent_queue_iterator_base::type, Allocator> { + using base_type = concurrent_queue_iterator_base::type, Allocator>; +public: + using value_type = Value; + using pointer = value_type*; + using reference = value_type&; + using difference_type = std::ptrdiff_t; + using iterator_category = std::forward_iterator_tag; + + concurrent_queue_iterator() = default; + + /** If Value==Container::value_type, then this routine is the copy constructor. + If Value==const Container::value_type, then this routine is a conversion constructor. */ + concurrent_queue_iterator( const concurrent_queue_iterator& other ) + : base_type(other) {} + +private: + concurrent_queue_iterator( const Container& container ) + : base_type(container.my_queue_representation) {} +public: + concurrent_queue_iterator& operator=( const concurrent_queue_iterator& other ) { + this->assign(other); + return *this; + } + + reference operator*() const { + return *static_cast(this->my_item); + } + + pointer operator->() const { return &operator*(); } + + concurrent_queue_iterator& operator++() { + this->advance(); + return *this; + } + + concurrent_queue_iterator operator++(int) { + concurrent_queue_iterator tmp = *this; + ++*this; + return tmp; + } + + friend bool operator==( const concurrent_queue_iterator& lhs, const concurrent_queue_iterator& rhs ) { + return lhs.my_item == rhs.my_item; + } + + friend bool operator!=( const concurrent_queue_iterator& lhs, const concurrent_queue_iterator& rhs ) { + return lhs.my_item != rhs.my_item; + } +private: + friend struct concurrent_queue_iterator_provider; +}; // class concurrent_queue_iterator + +} // namespace d2 +} // namespace detail +} // tbb + +#endif // __TBB_detail__concurrent_queue_base_H diff --git a/src/3rd party/oneapi/tbb/detail/_concurrent_skip_list.h b/src/3rd party/oneapi/tbb/detail/_concurrent_skip_list.h new file mode 100644 index 0000000000..2448bf042c --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_concurrent_skip_list.h @@ -0,0 +1,1352 @@ +/* + Copyright (c) 2019-2025 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__concurrent_skip_list_H +#define __TBB_detail__concurrent_skip_list_H + +#if !defined(__TBB_concurrent_map_H) && !defined(__TBB_concurrent_set_H) +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +#include "_config.h" +#include "_range_common.h" +#include "_allocator_traits.h" +#include "_template_helpers.h" +#include "_node_handle.h" +#include "_containers_helpers.h" +#include "_assert.h" +#include "_exception.h" +#include "../enumerable_thread_specific.h" +#include +#include +#include +#include +#include +#include // Need std::geometric_distribution +#include // Need std::equal and std::lexicographical_compare +#include +#if __TBB_CPP20_COMPARISONS_PRESENT +#include +#endif + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(push) +#pragma warning(disable: 4127) // warning C4127: conditional expression is constant +#endif + +namespace tbb { +namespace detail { +namespace d3 { + +template +class skip_list_thread_data { +public: + using level_generator_type = LevelGenerator; + using size_type = SizeType; + + skip_list_thread_data() : my_local_size(0) {} + + std::size_t random_level() { return my_rng(); } + std::size_t local_size() const { return my_local_size.load(std::memory_order_relaxed); } + + void set_size(size_type size) { my_local_size.store(size, std::memory_order_relaxed); } + void increment_size() { + // Not using my_local_size.fetch_add(1) to avoid exclusive locking of the cache line + // Since only one thread can write to this atomic, and no synchronization needed with readers + my_local_size.store(local_size() + 1, std::memory_order_relaxed); + } + +private: + level_generator_type my_rng; + // Atomic to avoid formal data race between several readers and single writer + std::atomic my_local_size; +}; + +template +class skip_list_node { + using node_ptr = skip_list_node*; +public: + using value_type = Value; + using atomic_node_ptr = std::atomic; + using size_type = std::size_t; + using container_allocator_type = Allocator; + + using reference = value_type&; + using const_reference = const value_type&; +private: + using allocator_traits = tbb::detail::allocator_traits; + + // Allocator is the same as the container allocator=> allocates unitptr_t + // It is required to rebind it to value_type to get the correct pointer and const_pointer + using value_allocator_traits = typename allocator_traits::template rebind_traits; +public: + using pointer = typename value_allocator_traits::pointer; + using const_pointer = typename value_allocator_traits::const_pointer; + + //In perfect world these constructor and destructor would have been private, + //however this seems technically impractical due to use of allocator_traits. + + //Should not be called directly, instead use create method + skip_list_node( size_type levels ) + : my_height(levels), my_index_number(0) + {} + + //Should not be called directly, instead use destroy method + ~skip_list_node() {} + + skip_list_node( const skip_list_node& ) = delete; + skip_list_node( skip_list_node&& ) = delete; + skip_list_node& operator=( const skip_list_node& ) = delete; + skip_list_node& operator=( skip_list_node&& ) = delete; + + static skip_list_node* create( container_allocator_type& alloc, size_type height ) { + size_type sz = calc_node_size(height); + static_assert(std::is_same::value, "skip_list_node assumes that passed in allocator operates on bytes"); + auto* node = reinterpret_cast(allocator_traits::allocate(alloc, sz)); + + //Construct the node itself + allocator_traits::construct(alloc, node, height); + + //Construct the level pointers + for (size_type l = 0; l < height; ++l) { + allocator_traits::construct(alloc, &node->get_atomic_next(l), nullptr); + } + + return node; + } + + static void destroy( container_allocator_type& alloc, skip_list_node* node ) { + //Destroy the level pointers + for (size_type l = 0; l < node->height(); ++l) { + allocator_traits::destroy(alloc, &node->atomic_next(l)); + } + size_type sz = calc_node_size(node->height()); + // Destroy the node itself + allocator_traits::destroy(alloc, node); + + // Deallocate the node + allocator_traits::deallocate(alloc, reinterpret_cast(node), sz); + } + + + pointer storage() { + return &my_value; + } + + reference value() { + return *storage(); + } + + node_ptr next( size_type level ) const { + node_ptr res = get_atomic_next(level).load(std::memory_order_acquire); + __TBB_ASSERT(res == nullptr || res->height() > level, "Broken internal structure"); + return res; + } + + atomic_node_ptr& atomic_next( size_type level ) { + atomic_node_ptr& res = get_atomic_next(level); +#if TBB_USE_DEBUG + node_ptr node = res.load(std::memory_order_acquire); + __TBB_ASSERT(node == nullptr || node->height() > level, "Broken internal structure"); +#endif + return res; + } + + void set_next( size_type level, node_ptr n ) { + __TBB_ASSERT(n == nullptr || n->height() > level, "Broken internal structure"); + get_atomic_next(level).store(n, std::memory_order_relaxed); + } + + size_type height() const { + return my_height; + } + + void set_index_number( size_type index_num ) { + my_index_number = index_num; + } + + size_type index_number() const { + return my_index_number; + } + +private: + static size_type calc_node_size( size_type height ) { + static_assert(alignof(skip_list_node) >= alignof(atomic_node_ptr), "Incorrect alignment"); + return sizeof(skip_list_node) + height * sizeof(atomic_node_ptr); + } + + atomic_node_ptr& get_atomic_next( size_type level ) { + atomic_node_ptr* arr = reinterpret_cast(this + 1); + return arr[level]; + } + + const atomic_node_ptr& get_atomic_next( size_type level ) const { + const atomic_node_ptr* arr = reinterpret_cast(this + 1); + return arr[level]; + } + + union { + value_type my_value; + }; + size_type my_height; + size_type my_index_number; +}; // class skip_list_node + +template +class skip_list_iterator { + using node_type = NodeType; + using node_ptr = node_type*; +public: + using iterator_category = std::forward_iterator_tag; + using value_type = ValueType; + + using difference_type = std::ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + skip_list_iterator() : skip_list_iterator(nullptr) {} + + skip_list_iterator( const skip_list_iterator& other ) + : my_node_ptr(other.my_node_ptr) {} + + skip_list_iterator& operator=( const skip_list_iterator& other ) { + my_node_ptr = other.my_node_ptr; + return *this; + } + + reference operator*() const { return my_node_ptr->value(); } + pointer operator->() const { return my_node_ptr->storage(); } + + skip_list_iterator& operator++() { + __TBB_ASSERT(my_node_ptr != nullptr, nullptr); + my_node_ptr = my_node_ptr->next(0); + return *this; + } + + skip_list_iterator operator++(int) { + skip_list_iterator tmp = *this; + ++*this; + return tmp; + } + +private: + skip_list_iterator(node_type* n) : my_node_ptr(n) {} + + node_ptr my_node_ptr; + + template + friend class concurrent_skip_list; + + template + friend class skip_list_iterator; + + friend class const_range; + friend class range; + + friend bool operator==( const skip_list_iterator& lhs, const skip_list_iterator& rhs ) { + return lhs.my_node_ptr == rhs.my_node_ptr; + } + + friend bool operator!=( const skip_list_iterator& lhs, const skip_list_iterator& rhs ) { + return lhs.my_node_ptr != rhs.my_node_ptr; + } +}; // class skip_list_iterator + +template +class concurrent_skip_list { +protected: + using container_traits = Traits; + using self_type = concurrent_skip_list; + using allocator_type = typename container_traits::allocator_type; + using allocator_traits_type = tbb::detail::allocator_traits; + using key_compare = typename container_traits::compare_type; + using value_compare = typename container_traits::value_compare; + using key_type = typename container_traits::key_type; + using value_type = typename container_traits::value_type; + static_assert(std::is_same::value, + "value_type of the container should be the same as its allocator"); + + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + + static constexpr size_type max_level = container_traits::max_level; + + using node_allocator_type = typename allocator_traits_type::template rebind_alloc; + using node_allocator_traits = tbb::detail::allocator_traits; + + using list_node_type = skip_list_node; + using atomic_node_ptr = std::atomic; + using node_type = d1::node_handle; + + using iterator = skip_list_iterator; + using const_iterator = skip_list_iterator; + + using reference = value_type&; + using const_reference = const value_type&; + using pointer = typename allocator_traits_type::pointer; + using const_pointer = typename allocator_traits_type::const_pointer; + + using random_level_generator_type = typename container_traits::random_level_generator_type; + using thread_data_type = skip_list_thread_data; +#if __TBB_RESUMABLE_TASKS + using ets_type = tbb::enumerable_thread_specific, + ets_suspend_aware>; +#else + using ets_type = tbb::enumerable_thread_specific; +#endif + + using node_ptr = list_node_type*; + + using array_type = std::array; +private: + template + using is_transparent = dependent_bool, T>; +public: + static constexpr bool allow_multimapping = container_traits::allow_multimapping; + + concurrent_skip_list() + : my_head_ptr(nullptr) + , my_max_height(0) {} + + explicit concurrent_skip_list( const key_compare& comp, const allocator_type& alloc = allocator_type() ) + : my_node_allocator(alloc), my_compare(comp), my_head_ptr(nullptr) + , my_max_height(0) {} + + explicit concurrent_skip_list( const allocator_type& alloc ) + : concurrent_skip_list(key_compare(), alloc) {} + + template + concurrent_skip_list( InputIterator first, InputIterator last, const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type() ) + : concurrent_skip_list(comp, alloc) + { + internal_copy(first, last); + } + + template + concurrent_skip_list( InputIterator first, InputIterator last, const allocator_type& alloc ) + : concurrent_skip_list(first, last, key_compare(), alloc) {} + + concurrent_skip_list( std::initializer_list init, const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type() ) + : concurrent_skip_list(init.begin(), init.end(), comp, alloc) {} + + concurrent_skip_list( std::initializer_list init, const allocator_type& alloc ) + : concurrent_skip_list(init, key_compare(), alloc) {} + + concurrent_skip_list( const concurrent_skip_list& other ) + : my_node_allocator(node_allocator_traits::select_on_container_copy_construction(other.get_allocator())) + , my_compare(other.my_compare) + , my_head_ptr(nullptr) + , my_max_height(0) + { + internal_copy(other); + __TBB_ASSERT(size() == other.size(), "Wrong size of copy-constructed container"); + } + + concurrent_skip_list( const concurrent_skip_list& other, const allocator_type& alloc ) + : my_node_allocator(alloc), my_compare(other.my_compare) + , my_head_ptr(nullptr) + , my_max_height(0) + { + internal_copy(other); + __TBB_ASSERT(size() == other.size(), "Wrong size of copy-constructed container"); + } + + concurrent_skip_list( concurrent_skip_list&& other ) + : my_node_allocator(std::move(other.my_node_allocator)) + , my_compare(other.my_compare) + , my_head_ptr(nullptr) // my_head_ptr would be stored in internal_move + { + internal_move(std::move(other)); + } + + concurrent_skip_list( concurrent_skip_list&& other, const allocator_type& alloc ) + : my_node_allocator(alloc) + , my_compare(other.my_compare) + , my_head_ptr(nullptr) + { + using is_always_equal = typename allocator_traits_type::is_always_equal; + internal_move_construct_with_allocator(std::move(other), is_always_equal()); + } + + ~concurrent_skip_list() { + clear(); + delete_head(); + } + + concurrent_skip_list& operator=( const concurrent_skip_list& other ) { + if (this != &other) { + clear(); + copy_assign_allocators(my_node_allocator, other.my_node_allocator); + my_compare = other.my_compare; + internal_copy(other); + } + return *this; + } + + concurrent_skip_list& operator=( concurrent_skip_list&& other ) { + if (this != &other) { + clear(); + delete_head(); + + my_compare = std::move(other.my_compare); + move_assign_allocators(my_node_allocator, other.my_node_allocator); + using pocma_type = typename node_allocator_traits::propagate_on_container_move_assignment; + using is_always_equal = typename node_allocator_traits::is_always_equal; + internal_move_assign(std::move(other), tbb::detail::disjunction()); + } + return *this; + } + + concurrent_skip_list& operator=( std::initializer_list il ) + { + clear(); + insert(il.begin(),il.end()); + return *this; + } + + std::pair insert( const value_type& value ) { + return internal_insert(value); + } + + std::pair insert( value_type&& value ) { + return internal_insert(std::move(value)); + } + + iterator insert( const_iterator, const_reference value ) { + // Ignore hint + return insert(value).first; + } + + iterator insert( const_iterator, value_type&& value ) { + // Ignore hint + return insert(std::move(value)).first; + } + + template + void insert( InputIterator first, InputIterator last ) { + while (first != last) { + insert(*first); + ++first; + } + } + + void insert( std::initializer_list init ) { + insert(init.begin(), init.end()); + } + + std::pair insert( node_type&& nh ) { + if (!nh.empty()) { + auto insert_node = d1::node_handle_accessor::get_node_ptr(nh); + thread_data_type& td = my_ets.local(); + std::pair insert_result = internal_insert_node(insert_node, td); + if (insert_result.second) { + d1::node_handle_accessor::deactivate(nh); + } + return insert_result; + } + return std::pair(end(), false); + } + + iterator insert( const_iterator, node_type&& nh ) { + // Ignore hint + return insert(std::move(nh)).first; + } + + template + std::pair emplace( Args&&... args ) { + return internal_insert(std::forward(args)...); + } + + template + iterator emplace_hint( const_iterator, Args&&... args ) { + // Ignore hint + return emplace(std::forward(args)...).first; + } + + iterator unsafe_erase( iterator pos ) { + std::pair extract_result = internal_extract(pos); + if (extract_result.first) { // node was extracted + delete_value_node(extract_result.first); + return extract_result.second; + } + return end(); + } + + iterator unsafe_erase( const_iterator pos ) { + return unsafe_erase(get_iterator(pos)); + } + + iterator unsafe_erase( const_iterator first, const_iterator last ) { + while (first != last) { + // Unsafe erase returns the iterator which follows the erased one + first = unsafe_erase(first); + } + return get_iterator(first); + } + + size_type unsafe_erase( const key_type& key ) { + return internal_erase(key); + } + + template + typename std::enable_if::value + && !std::is_convertible::value + && !std::is_convertible::value, + size_type>::type unsafe_erase( const K& key ) + { + return internal_erase(key); + } + + node_type unsafe_extract( const_iterator pos ) { + std::pair extract_result = internal_extract(pos); + return extract_result.first ? d1::node_handle_accessor::construct(extract_result.first) : node_type(); + } + + node_type unsafe_extract( iterator pos ) { + return unsafe_extract(const_iterator(pos)); + } + + node_type unsafe_extract( const key_type& key ) { + return unsafe_extract(find(key)); + } + + template + typename std::enable_if::value + && !std::is_convertible::value + && !std::is_convertible::value, + node_type>::type unsafe_extract( const K& key ) + { + return unsafe_extract(find(key)); + } + + iterator lower_bound( const key_type& key ) { + return iterator(internal_get_bound(key, my_compare)); + } + + const_iterator lower_bound( const key_type& key ) const { + return const_iterator(internal_get_bound(key, my_compare)); + } + + template + typename std::enable_if::value, iterator>::type lower_bound( const K& key ) { + return iterator(internal_get_bound(key, my_compare)); + } + + template + typename std::enable_if::value, const_iterator>::type lower_bound( const K& key ) const { + return const_iterator(internal_get_bound(key, my_compare)); + } + + iterator upper_bound( const key_type& key ) { + return iterator(internal_get_bound(key, not_greater_compare(my_compare))); + } + + const_iterator upper_bound( const key_type& key ) const { + return const_iterator(internal_get_bound(key, not_greater_compare(my_compare))); + } + + template + typename std::enable_if::value, iterator>::type upper_bound( const K& key ) { + return iterator(internal_get_bound(key, not_greater_compare(my_compare))); + } + + template + typename std::enable_if::value, const_iterator>::type upper_bound( const K& key ) const { + return const_iterator(internal_get_bound(key, not_greater_compare(my_compare))); + } + + iterator find( const key_type& key ) { + return iterator(internal_find(key)); + } + + const_iterator find( const key_type& key ) const { + return const_iterator(internal_find(key)); + } + + template + typename std::enable_if::value, iterator>::type find( const K& key ) { + return iterator(internal_find(key)); + } + + template + typename std::enable_if::value, const_iterator>::type find( const K& key ) const { + return const_iterator(internal_find(key)); + } + + size_type count( const key_type& key ) const { + return internal_count(key); + } + + template + typename std::enable_if::value, size_type>::type count( const K& key ) const { + return internal_count(key); + } + + bool contains( const key_type& key ) const { + return find(key) != end(); + } + + template + typename std::enable_if::value, bool>::type contains( const K& key ) const { + return find(key) != end(); + } + + void clear() noexcept { + // clear is not thread safe - load can be relaxed + node_ptr head = my_head_ptr.load(std::memory_order_relaxed); + + if (head == nullptr) return; // Head is not allocated => container is empty + + node_ptr current = head->next(0); + + // Delete all value nodes in the container + while (current) { + node_ptr next = current->next(0); + delete_value_node(current); + current = next; + } + + for (size_type level = 0; level < head->height(); ++level) { + head->set_next(level, nullptr); + } + + my_ets.clear(); + my_max_height.store(0, std::memory_order_relaxed); + } + + iterator begin() { + return iterator(internal_begin()); + } + + const_iterator begin() const { + return const_iterator(internal_begin()); + } + + const_iterator cbegin() const { + return const_iterator(internal_begin()); + } + + iterator end() { + return iterator(nullptr); + } + + const_iterator end() const { + return const_iterator(nullptr); + } + + const_iterator cend() const { + return const_iterator(nullptr); + } + + size_type size() const { + size_type total_size = 0; + + my_ets.combine_each([&](const thread_data_type& td) { + total_size += td.local_size(); + }); + + return total_size; + } + + size_type max_size() const { + return node_allocator_traits::max_size(my_node_allocator); + } + + __TBB_nodiscard bool empty() const { + return 0 == size(); + } + + allocator_type get_allocator() const { + return my_node_allocator; + } + + void swap(concurrent_skip_list& other) { + if (this != &other) { + using pocs_type = typename node_allocator_traits::propagate_on_container_swap; + using is_always_equal = typename node_allocator_traits::is_always_equal; + internal_swap(other, tbb::detail::disjunction()); + } + } + + std::pair equal_range(const key_type& key) { + return internal_equal_range(key); + } + + std::pair equal_range(const key_type& key) const { + return internal_equal_range(key); + } + + template + typename std::enable_if::value, std::pair>::type equal_range( const K& key ) { + return internal_equal_range(key); + } + + template + typename std::enable_if::value, std::pair>::type equal_range( const K& key ) const { + return internal_equal_range(key); + } + + key_compare key_comp() const { return my_compare; } + + value_compare value_comp() const { return container_traits::value_comp(my_compare); } + + class const_range_type { + public: + using size_type = typename concurrent_skip_list::size_type; + using difference_type = typename concurrent_skip_list::difference_type; + using iterator = typename concurrent_skip_list::const_iterator; + using value_type = typename iterator::value_type; + using reference = typename iterator::reference; + + bool empty() const { + return my_begin.my_node_ptr ? (my_begin.my_node_ptr->next(0) == my_end.my_node_ptr) + : true; + } + + bool is_divisible() const { + return my_begin.my_node_ptr && my_level != 0 + ? my_begin.my_node_ptr->next(my_level - 1) != my_end.my_node_ptr + : false; + } + + size_type size() const { return std::distance(my_begin, my_end); } + + const_range_type( const_range_type& r, split) + : my_end(r.my_end) { + if (r.empty()) { + __TBB_ASSERT(my_end.my_node_ptr == nullptr, nullptr); + my_begin = my_end; + my_level = 0; + } else { + my_begin = iterator(r.my_begin.my_node_ptr->next(r.my_level - 1)); + my_level = my_begin.my_node_ptr->height(); + } + r.my_end = my_begin; + } + + const_range_type( const concurrent_skip_list& l) + : my_end(l.end()), my_begin(l.begin()), + my_level(my_begin.my_node_ptr ? my_begin.my_node_ptr->height() : 0) {} + + iterator begin() const { return my_begin; } + iterator end() const { return my_end; } + size_type grainsize() const { return 1; } + + private: + const_iterator my_end; + const_iterator my_begin; + size_type my_level; + }; // class const_range_type + + class range_type : public const_range_type { + public: + using iterator = typename concurrent_skip_list::iterator; + using value_type = typename iterator::value_type; + using reference = typename iterator::reference; + + range_type(range_type& r, split) : const_range_type(r, split()) {} + range_type(const concurrent_skip_list& l) : const_range_type(l) {} + + iterator begin() const { + node_ptr node = const_range_type::begin().my_node_ptr; + return iterator(node); + } + + iterator end() const { + node_ptr node = const_range_type::end().my_node_ptr; + return iterator(node); + } + }; // class range_type + + range_type range() { return range_type(*this); } + const_range_type range() const { return const_range_type(*this); } + +private: + node_ptr internal_begin() const { + node_ptr head = get_head(); + return head == nullptr ? head : head->next(0); + } + + void internal_move(concurrent_skip_list&& other) { + my_head_ptr.store(other.my_head_ptr.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.my_head_ptr.store(nullptr, std::memory_order_relaxed); + + // TODO: investigate whether the performance gain from shrinking sizes to calling thread's size is justified + // Logic similar to enumerable_thread_specific::internal_move can be used instead + my_ets.clear(); + my_ets.local().set_size(other.size()); + other.my_ets.clear(); + + my_max_height.store(other.my_max_height.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.my_max_height.store(0, std::memory_order_relaxed); + } + + void internal_move_construct_with_allocator(concurrent_skip_list&& other, + /*is_always_equal = */std::true_type) { + internal_move(std::move(other)); + } + + void internal_move_construct_with_allocator(concurrent_skip_list&& other, + /*is_always_equal = */std::false_type) { + if (my_node_allocator == other.get_allocator()) { + internal_move(std::move(other)); + } else { + my_ets.clear(); + my_ets.local().set_size(0); + + my_max_height.store(other.my_max_height.load(std::memory_order_relaxed), std::memory_order_relaxed); + internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); + } + } + + static const key_type& get_key( node_ptr n ) { + __TBB_ASSERT(n, nullptr); + return container_traits::get_key(static_cast(n)->value()); + } + + template + bool found( node_ptr node, const K& key ) const { + return node != nullptr && !my_compare(key, get_key(node)); + } + + template + node_ptr internal_find(const K& key) const { + return allow_multimapping ? internal_find_multi(key) : internal_find_unique(key); + } + + template + node_ptr internal_find_multi( const K& key ) const { + node_ptr prev = get_head(); + if (prev == nullptr) return nullptr; // If the head node is not allocated - exit + + node_ptr curr = nullptr; + node_ptr old_curr = curr; + + for (size_type h = my_max_height.load(std::memory_order_acquire); h > 0; --h) { + curr = internal_find_position(h - 1, prev, key, my_compare); + + if (curr != old_curr && found(curr, key)) { + return curr; + } + old_curr = curr; + } + return nullptr; + } + + template + node_ptr internal_find_unique( const K& key ) const { + const_iterator it = lower_bound(key); + return (it == end() || my_compare(key, container_traits::get_key(*it))) ? nullptr : it.my_node_ptr; + } + + template + size_type internal_count( const K& key ) const { + if (allow_multimapping) { + // TODO: reimplement without double traversal + std::pair r = equal_range(key); + return std::distance(r.first, r.second); + } + return size_type(contains(key) ? 1 : 0); + } + + template + std::pair internal_equal_range(const K& key) const { + iterator lb = get_iterator(lower_bound(key)); + auto result = std::make_pair(lb, lb); + + // If the lower bound points to the node with the requested key + if (found(lb.my_node_ptr, key)) { + + if (!allow_multimapping) { + // For unique containers - move the second iterator forward and exit + ++result.second; + } else { + // For multi containers - find the upper bound starting from the lower bound + node_ptr prev = lb.my_node_ptr; + node_ptr curr = nullptr; + not_greater_compare cmp(my_compare); + + // Start from the lower bound of the range + for (size_type h = prev->height(); h > 0; --h) { + curr = prev->next(h - 1); + while (curr && cmp(get_key(curr), key)) { + prev = curr; + // If the height of the next node is greater than the current one - jump to its height + if (h < curr->height()) { + h = curr->height(); + } + curr = prev->next(h - 1); + } + } + result.second = iterator(curr); + } + } + + return result; + } + + // Finds position on the level using comparator cmp starting from the node prev + template + node_ptr internal_find_position( size_type level, node_ptr& prev, const K& key, + const Comparator& cmp ) const { + __TBB_ASSERT(level < prev->height(), "Wrong level to find position"); + node_ptr curr = prev->next(level); + + while (curr && cmp(get_key(curr), key)) { + prev = curr; + __TBB_ASSERT(level < prev->height(), nullptr); + curr = prev->next(level); + } + + return curr; + } + + // The same as previous overload, but allows index_number comparison + template + node_ptr internal_find_position( size_type level, node_ptr& prev, node_ptr node, + const Comparator& cmp ) const { + __TBB_ASSERT(level < prev->height(), "Wrong level to find position"); + node_ptr curr = prev->next(level); + + while (curr && cmp(get_key(curr), get_key(node))) { + if (allow_multimapping && cmp(get_key(node), get_key(curr)) && curr->index_number() > node->index_number()) { + break; + } + + prev = curr; + __TBB_ASSERT(level < prev->height(), nullptr); + curr = prev->next(level); + } + return curr; + } + + template + void fill_prev_curr_arrays(array_type& prev_nodes, array_type& curr_nodes, node_ptr node, const key_type& key, + const Comparator& cmp, node_ptr head ) { + + size_type curr_max_height = my_max_height.load(std::memory_order_acquire); + size_type node_height = node->height(); + if (curr_max_height < node_height) { + std::fill(prev_nodes.begin() + curr_max_height, prev_nodes.begin() + node_height, head); + std::fill(curr_nodes.begin() + curr_max_height, curr_nodes.begin() + node_height, nullptr); + } + + node_ptr prev = head; + for (size_type level = curr_max_height; level > 0; --level) { + node_ptr curr = internal_find_position(level - 1, prev, key, cmp); + prev_nodes[level - 1] = prev; + curr_nodes[level - 1] = curr; + } + } + + void fill_prev_array_for_existing_node( array_type& prev_nodes, node_ptr node ) { + node_ptr head = create_head_if_necessary(); + prev_nodes.fill(head); + + node_ptr prev = head; + for (size_type level = node->height(); level > 0; --level) { + while (prev->next(level - 1) != node) { + prev = prev->next(level - 1); + } + prev_nodes[level - 1] = prev; + } + } + + struct not_greater_compare { + const key_compare& my_less_compare; + + not_greater_compare( const key_compare& less_compare ) : my_less_compare(less_compare) {} + + template + bool operator()( const K1& first, const K2& second ) const { + return !my_less_compare(second, first); + } + }; + + not_greater_compare select_comparator( /*allow_multimapping = */ std::true_type ) { + return not_greater_compare(my_compare); + } + + key_compare select_comparator( /*allow_multimapping = */ std::false_type ) { + return my_compare; + } + + template + std::pair internal_insert( Args&&... args ) { + thread_data_type& td = my_ets.local(); + node_ptr new_node = create_value_node(td, std::forward(args)...); + std::pair insert_result = internal_insert_node(new_node, td); + if (!insert_result.second) { + delete_value_node(new_node); + } + return insert_result; + } + + std::pair internal_insert_node( node_ptr new_node, thread_data_type& td ) + { + array_type prev_nodes; + array_type curr_nodes; + size_type new_height = new_node->height(); + auto compare = select_comparator(std::integral_constant{}); + + node_ptr head_node = create_head_if_necessary(); + + for (;;) { + fill_prev_curr_arrays(prev_nodes, curr_nodes, new_node, get_key(new_node), compare, head_node); + + node_ptr prev = prev_nodes[0]; + node_ptr next = curr_nodes[0]; + + if (allow_multimapping) { + new_node->set_index_number(prev->index_number() + 1); + } else { + if (found(next, get_key(new_node))) { + return std::pair(iterator(next), false); + } + } + + new_node->set_next(0, next); + if (!prev->atomic_next(0).compare_exchange_strong(next, new_node)) { + continue; + } + + // If the node was successfully linked on the first level - it will be linked on other levels + // Insertion cannot fail starting from this point + + // If the height of inserted node is greater than maximum - increase maximum + size_type max_height = my_max_height.load(std::memory_order_acquire); + for (;;) { + if (new_height <= max_height || my_max_height.compare_exchange_strong(max_height, new_height)) { + // If the maximum was successfully updated by current thread + // or by an other thread for the value, greater or equal to new_height + break; + } + } + + for (std::size_t level = 1; level < new_height; ++level) { + // Link the node on upper levels + for (;;) { + prev = prev_nodes[level]; + next = static_cast(curr_nodes[level]); + + new_node->set_next(level, next); + __TBB_ASSERT(new_node->height() > level, "Internal structure break"); + if (prev->atomic_next(level).compare_exchange_strong(next, new_node)) { + break; + } + + for (size_type lev = level; lev != new_height; ++lev ) { + curr_nodes[lev] = internal_find_position(lev, prev_nodes[lev], new_node, compare); + } + } + } + td.increment_size(); + return std::pair(iterator(new_node), true); + } + } + + template + node_ptr internal_get_bound( const K& key, const Comparator& cmp ) const { + node_ptr prev = get_head(); + if (prev == nullptr) return nullptr; // If the head node is not allocated - exit + + node_ptr curr = nullptr; + + for (size_type h = my_max_height.load(std::memory_order_acquire); h > 0; --h) { + curr = internal_find_position(h - 1, prev, key, cmp); + } + + return curr; + } + + template + size_type internal_erase( const K& key ) { + auto eq = equal_range(key); + size_type old_size = size(); + unsafe_erase(eq.first, eq.second); + return old_size - size(); + } + + // Returns node_ptr to the extracted node and node_ptr to the next node after the extracted + std::pair internal_extract( const_iterator it ) { + std::pair result(nullptr, nullptr); + if ( it != end() ) { + array_type prev_nodes; + + node_ptr erase_node = it.my_node_ptr; + node_ptr next_node = erase_node->next(0); + fill_prev_array_for_existing_node(prev_nodes, erase_node); + + for (size_type level = 0; level < erase_node->height(); ++level) { + prev_nodes[level]->set_next(level, erase_node->next(level)); + erase_node->set_next(level, nullptr); + } + + // Shrink all thread-local sizes into the calling thread's size and decrement + // TODO: investigate if performance gain of this optimization is justified + size_type total_size = size(); + my_ets.clear(); + my_ets.local().set_size(total_size - 1); + + result.first = erase_node; + result.second = next_node; + } + return result; + } + +protected: + template + void internal_merge( SourceType&& source ) { + using source_type = typename std::decay::type; + using source_iterator = typename source_type::iterator; + static_assert((std::is_same::value), "Incompatible containers cannot be merged"); + + for (source_iterator it = source.begin(); it != source.end();) { + source_iterator where = it++; + if (allow_multimapping || !contains(container_traits::get_key(*where))) { + node_type handle = source.unsafe_extract(where); + __TBB_ASSERT(!handle.empty(), "Extracted handle in merge is empty"); + + if (!insert(std::move(handle)).second) { + __TBB_ASSERT(!handle.empty(), "Handle should not be empty if insert fails"); + //If the insertion fails - return the node into source + source.insert(std::move(handle)); + } + __TBB_ASSERT(handle.empty(), "Node handle should be empty after the insertion"); + } + } + } + +private: + void internal_copy( const concurrent_skip_list& other ) { + internal_copy(other.begin(), other.end()); + } + + template + void internal_copy( Iterator first, Iterator last ) { + try_call([&] { + for (auto it = first; it != last; ++it) { + insert(*it); + } + }).on_exception([&] { + clear(); + delete_head(); + }); + } + + node_ptr create_node( size_type height ) { + return list_node_type::create(my_node_allocator, height); + } + + template + node_ptr create_value_node( thread_data_type& td, Args&&... args ) + { + node_ptr node = create_node(td.random_level()); + + // try_call API is not convenient here due to broken + // variadic capture on GCC 4.8.5 + auto value_guard = make_raii_guard([&] { + delete_node(node); + }); + + // Construct the value inside the node + node_allocator_traits::construct(my_node_allocator, node->storage(), std::forward(args)...); + value_guard.dismiss(); + return node; + } + + node_ptr create_head_node() { + return create_node(max_level); + } + + void delete_head() { + node_ptr head = my_head_ptr.load(std::memory_order_relaxed); + if (head != nullptr) { + delete_node(head); + my_head_ptr.store(nullptr, std::memory_order_relaxed); + } + } + + void delete_node( node_ptr node ) { + list_node_type::destroy(my_node_allocator, node); + } + + void delete_value_node( node_ptr node ) { + // Destroy the value inside the node + node_allocator_traits::destroy(my_node_allocator, node->storage()); + delete_node(node); + } + + node_ptr get_head() const { + return my_head_ptr.load(std::memory_order_acquire); + } + + node_ptr create_head_if_necessary() { + node_ptr current_head = get_head(); + if (current_head == nullptr) { + // Head node was not created - create it + node_ptr new_head = create_head_node(); + if (my_head_ptr.compare_exchange_strong(current_head, new_head)) { + current_head = new_head; + } else { + // If an other thread has already created the head node - destroy new_head + // current_head now points to the actual head node + delete_node(new_head); + } + } + __TBB_ASSERT(my_head_ptr.load(std::memory_order_relaxed) != nullptr, nullptr); + __TBB_ASSERT(current_head != nullptr, nullptr); + return current_head; + } + + static iterator get_iterator( const_iterator it ) { + return iterator(it.my_node_ptr); + } + + void internal_move_assign( concurrent_skip_list&& other, /*POCMA || is_always_equal =*/std::true_type ) { + internal_move(std::move(other)); + } + + void internal_move_assign( concurrent_skip_list&& other, /*POCMA || is_always_equal =*/std::false_type ) { + if (my_node_allocator == other.my_node_allocator) { + internal_move(std::move(other)); + } else { + internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); + } + } + + void internal_swap_fields( concurrent_skip_list& other ) { + using std::swap; + swap_allocators(my_node_allocator, other.my_node_allocator); + swap(my_compare, other.my_compare); + swap_atomics_relaxed(my_head_ptr, other.my_head_ptr); + swap_atomics_relaxed(my_max_height, other.my_max_height); + + // TODO: investigate whether the performance gain from shrinking sizes to calling thread's size is justified + // Logic similar to enumerable_thread_specific::internal_swap can be used instead + size_type total_size = size(); + my_ets.clear(); + my_ets.local().set_size(other.size()); + other.my_ets.clear(); + other.my_ets.local().set_size(total_size); + } + + void internal_swap( concurrent_skip_list& other, /*POCMA || is_always_equal =*/std::true_type ) { + internal_swap_fields(other); + } + + void internal_swap( concurrent_skip_list& other, /*POCMA || is_always_equal =*/std::false_type ) { + __TBB_ASSERT(my_node_allocator == other.my_node_allocator, "Swapping with unequal allocators is not allowed"); + internal_swap_fields(other); + } + + node_allocator_type my_node_allocator; + key_compare my_compare; + atomic_node_ptr my_head_ptr; + std::atomic my_max_height; + mutable ets_type my_ets; // mutable is needed since ets::combine_each is non-const + + template + friend class concurrent_skip_list; +}; // class concurrent_skip_list + +template +bool operator==( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) { + if (lhs.size() != rhs.size()) return false; +#if _MSC_VER + // Passing "unchecked" iterators to std::equal with 3 parameters + // causes compiler warnings. + // The workaround is to use overload with 4 parameters, which is + // available since C++14 - minimally supported version on MSVC + return std::equal(lhs.begin(), lhs.end(), rhs.begin(), rhs.end()); +#else + return std::equal(lhs.begin(), lhs.end(), rhs.begin()); +#endif +} + +#if !__TBB_CPP20_COMPARISONS_PRESENT +template +bool operator!=( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) { + return !(lhs == rhs); +} +#endif + +#if __TBB_CPP20_COMPARISONS_PRESENT && __TBB_CPP20_CONCEPTS_PRESENT +template +tbb::detail::synthesized_three_way_result +operator<=>( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) { + return std::lexicographical_compare_three_way(lhs.begin(), lhs.end(), + rhs.begin(), rhs.end(), + tbb::detail::synthesized_three_way_comparator{}); +} +#else +template +bool operator<( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) { + return std::lexicographical_compare(lhs.begin(), lhs.end(), rhs.begin(), rhs.end()); +} + +template +bool operator>( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) { + return rhs < lhs; +} + +template +bool operator<=( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) { + return !(rhs < lhs); +} + +template +bool operator>=( const concurrent_skip_list& lhs, const concurrent_skip_list& rhs ) { + return !(lhs < rhs); +} +#endif // __TBB_CPP20_COMPARISONS_PRESENT && __TBB_CPP20_CONCEPTS_PRESENT + +// Generates a number from the interval [0, MaxLevel). +template +class geometric_level_generator { +public: + static constexpr std::size_t max_level = MaxLevel; + // TODO: modify the algorithm to accept other values of max_level + static_assert(max_level == 32, "Incompatible max_level for rng"); + + geometric_level_generator() : my_engine(std::minstd_rand::result_type(time(nullptr))) {} + + std::size_t operator()() { + // +1 is required to pass at least 1 into log2 (log2(0) is undefined) + // -1 is required to have an ability to return 0 from the generator (max_level - log2(2^31) - 1) + auto random_number = my_engine(); + + std::size_t result = max_level - std::size_t(tbb::detail::log2(random_number + 1)) - 1; + __TBB_ASSERT(result <= max_level, nullptr); + return result; + } +private: + std::minstd_rand my_engine; +}; + +} // namespace d3 + +} // namespace detail +} // namespace tbb + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(pop) // warning 4127 is back +#endif + +#endif // __TBB_detail__concurrent_skip_list_H diff --git a/src/3rd party/oneapi/tbb/detail/_concurrent_unordered_base.h b/src/3rd party/oneapi/tbb/detail/_concurrent_unordered_base.h new file mode 100644 index 0000000000..83c514987a --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_concurrent_unordered_base.h @@ -0,0 +1,1516 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + Copyright (c) 2025 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__concurrent_unordered_base_H +#define __TBB_detail__concurrent_unordered_base_H + +#if !defined(__TBB_concurrent_unordered_map_H) && !defined(__TBB_concurrent_unordered_set_H) +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +#include "_range_common.h" +#include "_containers_helpers.h" +#include "_segment_table.h" +#include "_hash_compare.h" +#include "_allocator_traits.h" +#include "_node_handle.h" +#include "_assert.h" +#include "_utils.h" +#include "_exception.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(push) +#pragma warning(disable: 4127) // warning C4127: conditional expression is constant +#endif + +namespace tbb { +namespace detail { +namespace d2 { + +template +class concurrent_unordered_base; + +template +class solist_iterator { +private: + using node_ptr = typename Container::value_node_ptr; + template + friend class split_ordered_list; + template + friend class solist_iterator; + template + friend class concurrent_unordered_base; + template + friend bool operator==( const solist_iterator& i, const solist_iterator& j ); + template + friend bool operator!=( const solist_iterator& i, const solist_iterator& j ); +public: + using value_type = Value; + using difference_type = typename Container::difference_type; + using pointer = value_type*; + using reference = value_type&; + using iterator_category = std::forward_iterator_tag; + + solist_iterator() : my_node_ptr(nullptr) {} + solist_iterator( const solist_iterator& other ) + : my_node_ptr(other.my_node_ptr) {} + + solist_iterator& operator=( const solist_iterator& other ) { + my_node_ptr = other.my_node_ptr; + return *this; + } + + reference operator*() const { + return my_node_ptr->value(); + } + + pointer operator->() const { + return my_node_ptr->storage(); + } + + solist_iterator& operator++() { + auto next_node = my_node_ptr->next(); + while(next_node && next_node->is_dummy()) { + next_node = next_node->next(); + } + my_node_ptr = static_cast(next_node); + return *this; + } + + solist_iterator operator++(int) { + solist_iterator tmp = *this; + ++*this; + return tmp; + } + +private: + solist_iterator( node_ptr pnode ) : my_node_ptr(pnode) {} + + node_ptr get_node_ptr() const { return my_node_ptr; } + + node_ptr my_node_ptr; +}; + +template +bool operator==( const solist_iterator& i, const solist_iterator& j ) { + return i.my_node_ptr == j.my_node_ptr; +} + +template +bool operator!=( const solist_iterator& i, const solist_iterator& j ) { + return i.my_node_ptr != j.my_node_ptr; +} + +template +class list_node { +public: + using node_ptr = list_node*; + using sokey_type = SokeyType; + + list_node(sokey_type key) : my_next(nullptr), my_order_key(key) {} + + void init( sokey_type key ) { + my_order_key = key; + } + + sokey_type order_key() const { + return my_order_key; + } + + bool is_dummy() { + // The last bit of order key is unset for dummy nodes + return (my_order_key & 0x1) == 0; + } + + node_ptr next() const { + return my_next.load(std::memory_order_acquire); + } + + void set_next( node_ptr next_node ) { + my_next.store(next_node, std::memory_order_release); + } + + bool try_set_next( node_ptr expected_next, node_ptr new_next ) { + return my_next.compare_exchange_strong(expected_next, new_next); + } + +private: + std::atomic my_next; + sokey_type my_order_key; +}; // class list_node + +template +class value_node : public list_node +{ +public: + using base_type = list_node; + using sokey_type = typename base_type::sokey_type; + using value_type = ValueType; + + value_node( sokey_type ord_key ) : base_type(ord_key) {} + ~value_node() {} + value_type* storage() { + return &my_value; + } + + value_type& value() { + return *storage(); + } + +private: + union { + value_type my_value; + }; +}; // class value_node + +template +class concurrent_unordered_base { + using self_type = concurrent_unordered_base; + using traits_type = Traits; + using hash_compare_type = typename traits_type::hash_compare_type; + class unordered_segment_table; +public: + using value_type = typename traits_type::value_type; + using key_type = typename traits_type::key_type; + using allocator_type = typename traits_type::allocator_type; + +private: + using allocator_traits_type = tbb::detail::allocator_traits; + // TODO: check assert conditions for different C++ standards + static_assert(std::is_same::value, + "value_type of the container must be the same as its allocator"); + using sokey_type = std::size_t; + +public: + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + + using iterator = solist_iterator; + using const_iterator = solist_iterator; + using local_iterator = iterator; + using const_local_iterator = const_iterator; + + using reference = value_type&; + using const_reference = const value_type&; + using pointer = typename allocator_traits_type::pointer; + using const_pointer = typename allocator_traits_type::const_pointer; + + using hasher = typename hash_compare_type::hasher; + using key_equal = typename hash_compare_type::key_equal; + +private: + using list_node_type = list_node; + using value_node_type = value_node; + using node_ptr = list_node_type*; + using value_node_ptr = value_node_type*; + + using value_node_allocator_type = typename allocator_traits_type::template rebind_alloc; + using node_allocator_type = typename allocator_traits_type::template rebind_alloc; + + using node_allocator_traits = tbb::detail::allocator_traits; + using value_node_allocator_traits = tbb::detail::allocator_traits; + + static constexpr size_type round_up_to_power_of_two( size_type bucket_count ) { + return size_type(1) << size_type(tbb::detail::log2(uintptr_t(bucket_count == 0 ? 1 : bucket_count) * 2 - 1)); + } + + template + using is_transparent = dependent_bool, T>; +public: + using node_type = d1::node_handle; + + explicit concurrent_unordered_base( size_type bucket_count, const hasher& hash = hasher(), + const key_equal& equal = key_equal(), const allocator_type& alloc = allocator_type() ) + : my_size(0), + my_bucket_count(round_up_to_power_of_two(bucket_count)), + my_max_load_factor(float(initial_max_load_factor)), + my_hash_compare(hash, equal), + my_head(sokey_type(0)), + my_segments(alloc) {} + + concurrent_unordered_base() : concurrent_unordered_base(initial_bucket_count) {} + + concurrent_unordered_base( size_type bucket_count, const allocator_type& alloc ) + : concurrent_unordered_base(bucket_count, hasher(), key_equal(), alloc) {} + + concurrent_unordered_base( size_type bucket_count, const hasher& hash, const allocator_type& alloc ) + : concurrent_unordered_base(bucket_count, hash, key_equal(), alloc) {} + + explicit concurrent_unordered_base( const allocator_type& alloc ) + : concurrent_unordered_base(initial_bucket_count, hasher(), key_equal(), alloc) {} + + template + concurrent_unordered_base( InputIterator first, InputIterator last, + size_type bucket_count = initial_bucket_count, const hasher& hash = hasher(), + const key_equal& equal = key_equal(), const allocator_type& alloc = allocator_type() ) + : concurrent_unordered_base(bucket_count, hash, equal, alloc) + { + insert(first, last); + } + + template + concurrent_unordered_base( InputIterator first, InputIterator last, + size_type bucket_count, const allocator_type& alloc ) + : concurrent_unordered_base(first, last, bucket_count, hasher(), key_equal(), alloc) {} + + template + concurrent_unordered_base( InputIterator first, InputIterator last, + size_type bucket_count, const hasher& hash, const allocator_type& alloc ) + : concurrent_unordered_base(first, last, bucket_count, hash, key_equal(), alloc) {} + + concurrent_unordered_base( const concurrent_unordered_base& other ) + : my_size(other.my_size.load(std::memory_order_relaxed)), + my_bucket_count(other.my_bucket_count.load(std::memory_order_relaxed)), + my_max_load_factor(other.my_max_load_factor), + my_hash_compare(other.my_hash_compare), + my_head(other.my_head.order_key()), + my_segments(other.my_segments) + { + try_call( [&] { + internal_copy(other); + } ).on_exception( [&] { + clear(); + }); + } + + concurrent_unordered_base( const concurrent_unordered_base& other, const allocator_type& alloc ) + : my_size(other.my_size.load(std::memory_order_relaxed)), + my_bucket_count(other.my_bucket_count.load(std::memory_order_relaxed)), + my_max_load_factor(other.my_max_load_factor), + my_hash_compare(other.my_hash_compare), + my_head(other.my_head.order_key()), + my_segments(other.my_segments, alloc) + { + try_call( [&] { + internal_copy(other); + } ).on_exception( [&] { + clear(); + }); + } + + concurrent_unordered_base( concurrent_unordered_base&& other ) + : my_size(other.my_size.load(std::memory_order_relaxed)), + my_bucket_count(other.my_bucket_count.load(std::memory_order_relaxed)), + my_max_load_factor(std::move(other.my_max_load_factor)), + my_hash_compare(std::move(other.my_hash_compare)), + my_head(other.my_head.order_key()), + my_segments(std::move(other.my_segments)) + { + move_content(std::move(other)); + } + + concurrent_unordered_base( concurrent_unordered_base&& other, const allocator_type& alloc ) + : my_size(other.my_size.load(std::memory_order_relaxed)), + my_bucket_count(other.my_bucket_count.load(std::memory_order_relaxed)), + my_max_load_factor(std::move(other.my_max_load_factor)), + my_hash_compare(std::move(other.my_hash_compare)), + my_head(other.my_head.order_key()), + my_segments(std::move(other.my_segments), alloc) + { + using is_always_equal = typename allocator_traits_type::is_always_equal; + internal_move_construct_with_allocator(std::move(other), alloc, is_always_equal()); + } + + concurrent_unordered_base( std::initializer_list init, + size_type bucket_count = initial_bucket_count, + const hasher& hash = hasher(), const key_equal& equal = key_equal(), + const allocator_type& alloc = allocator_type() ) + : concurrent_unordered_base(init.begin(), init.end(), bucket_count, hash, equal, alloc) {} + + concurrent_unordered_base( std::initializer_list init, + size_type bucket_count, const allocator_type& alloc ) + : concurrent_unordered_base(init, bucket_count, hasher(), key_equal(), alloc) {} + + concurrent_unordered_base( std::initializer_list init, + size_type bucket_count, const hasher& hash, const allocator_type& alloc ) + : concurrent_unordered_base(init, bucket_count, hash, key_equal(), alloc) {} + + ~concurrent_unordered_base() { + internal_clear(); + } + + concurrent_unordered_base& operator=( const concurrent_unordered_base& other ) { + if (this != &other) { + clear(); + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + my_bucket_count.store(other.my_bucket_count.load(std::memory_order_relaxed), std::memory_order_relaxed); + my_max_load_factor = other.my_max_load_factor; + my_hash_compare = other.my_hash_compare; + my_segments = other.my_segments; + internal_copy(other); // TODO: guards for exceptions? + } + return *this; + } + + concurrent_unordered_base& operator=( concurrent_unordered_base&& other ) noexcept(unordered_segment_table::is_noexcept_assignment) { + if (this != &other) { + clear(); + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + my_bucket_count.store(other.my_bucket_count.load(std::memory_order_relaxed), std::memory_order_relaxed); + my_max_load_factor = std::move(other.my_max_load_factor); + my_hash_compare = std::move(other.my_hash_compare); + my_segments = std::move(other.my_segments); + + using pocma_type = typename allocator_traits_type::propagate_on_container_move_assignment; + using is_always_equal = typename allocator_traits_type::is_always_equal; + internal_move_assign(std::move(other), tbb::detail::disjunction()); + } + return *this; + } + + concurrent_unordered_base& operator=( std::initializer_list init ) { + clear(); + insert(init); + return *this; + } + + void swap( concurrent_unordered_base& other ) noexcept(unordered_segment_table::is_noexcept_swap) { + if (this != &other) { + using pocs_type = typename allocator_traits_type::propagate_on_container_swap; + using is_always_equal = typename allocator_traits_type::is_always_equal; + internal_swap(other, tbb::detail::disjunction()); + } + } + + allocator_type get_allocator() const noexcept { return my_segments.get_allocator(); } + + iterator begin() noexcept { return iterator(first_value_node(&my_head)); } + const_iterator begin() const noexcept { return const_iterator(first_value_node(const_cast(&my_head))); } + const_iterator cbegin() const noexcept { return const_iterator(first_value_node(const_cast(&my_head))); } + + iterator end() noexcept { return iterator(nullptr); } + const_iterator end() const noexcept { return const_iterator(nullptr); } + const_iterator cend() const noexcept { return const_iterator(nullptr); } + + __TBB_nodiscard bool empty() const noexcept { return size() == 0; } + size_type size() const noexcept { return my_size.load(std::memory_order_relaxed); } + size_type max_size() const noexcept { return allocator_traits_type::max_size(get_allocator()); } + + void clear() noexcept { + internal_clear(); + } + + std::pair insert( const value_type& value ) { + return internal_insert_value(value); + } + + std::pair insert( value_type&& value ) { + return internal_insert_value(std::move(value)); + } + + iterator insert( const_iterator, const value_type& value ) { + // Ignore hint + return insert(value).first; + } + + iterator insert( const_iterator, value_type&& value ) { + // Ignore hint + return insert(std::move(value)).first; + } + + template + void insert( InputIterator first, InputIterator last ) { + for (; first != last; ++first) { + insert(*first); + } + } + + void insert( std::initializer_list init ) { + insert(init.begin(), init.end()); + } + + std::pair insert( node_type&& nh ) { + if (!nh.empty()) { + value_node_ptr insert_node = d1::node_handle_accessor::get_node_ptr(nh); + auto init_node = [&insert_node]( sokey_type order_key )->value_node_ptr { + insert_node->init(order_key); + return insert_node; + }; + auto insert_result = internal_insert(insert_node->value(), init_node); + if (insert_result.inserted) { + // If the insertion succeeded - set node handle to the empty state + __TBB_ASSERT(insert_result.remaining_node == nullptr, + "internal_insert_node should not return the remaining node if the insertion succeeded"); + d1::node_handle_accessor::deactivate(nh); + } + return { iterator(insert_result.node_with_equal_key), insert_result.inserted }; + } + return {end(), false}; + } + + iterator insert( const_iterator, node_type&& nh ) { + // Ignore hint + return insert(std::move(nh)).first; + } + + template + std::pair emplace( Args&&... args ) { + // Create a node with temporary order_key 0, which will be reinitialize + // in internal_insert after the hash calculation + value_node_ptr insert_node = create_node(0, std::forward(args)...); + + auto init_node = [&insert_node]( sokey_type order_key )->value_node_ptr { + insert_node->init(order_key); + return insert_node; + }; + + auto insert_result = internal_insert(insert_node->value(), init_node); + + if (!insert_result.inserted) { + // If the insertion failed - destroy the node which was created + insert_node->init(split_order_key_regular(1)); + destroy_node(insert_node); + } + + return { iterator(insert_result.node_with_equal_key), insert_result.inserted }; + } + + template + iterator emplace_hint( const_iterator, Args&&... args ) { + // Ignore hint + return emplace(std::forward(args)...).first; + } + + iterator unsafe_erase( const_iterator pos ) { + return iterator(first_value_node(internal_erase(pos.get_node_ptr()))); + } + + iterator unsafe_erase( iterator pos ) { + return iterator(first_value_node(internal_erase(pos.get_node_ptr()))); + } + + iterator unsafe_erase( const_iterator first, const_iterator last ) { + while(first != last) { + first = unsafe_erase(first); + } + return iterator(first.get_node_ptr()); + } + + size_type unsafe_erase( const key_type& key ) { + return internal_erase_by_key(key); + } + + template + typename std::enable_if::value + && !std::is_convertible::value + && !std::is_convertible::value, + size_type>::type unsafe_erase( const K& key ) + { + return internal_erase_by_key(key); + } + + node_type unsafe_extract( const_iterator pos ) { + internal_extract(pos.get_node_ptr()); + return d1::node_handle_accessor::construct(pos.get_node_ptr()); + } + + node_type unsafe_extract( iterator pos ) { + internal_extract(pos.get_node_ptr()); + return d1::node_handle_accessor::construct(pos.get_node_ptr()); + } + + node_type unsafe_extract( const key_type& key ) { + iterator item = find(key); + return item == end() ? node_type() : unsafe_extract(item); + } + + template + typename std::enable_if::value + && !std::is_convertible::value + && !std::is_convertible::value, + node_type>::type unsafe_extract( const K& key ) + { + iterator item = find(key); + return item == end() ? node_type() : unsafe_extract(item); + } + + // Lookup functions + iterator find( const key_type& key ) { + value_node_ptr result = internal_find(key); + return result == nullptr ? end() : iterator(result); + } + + const_iterator find( const key_type& key ) const { + value_node_ptr result = const_cast(this)->internal_find(key); + return result == nullptr ? end() : const_iterator(result); + } + + template + typename std::enable_if::value, iterator>::type find( const K& key ) { + value_node_ptr result = internal_find(key); + return result == nullptr ? end() : iterator(result); + } + + template + typename std::enable_if::value, const_iterator>::type find( const K& key ) const { + value_node_ptr result = const_cast(this)->internal_find(key); + return result == nullptr ? end() : const_iterator(result); + } + + std::pair equal_range( const key_type& key ) { + auto result = internal_equal_range(key); + return std::make_pair(iterator(result.first), iterator(result.second)); + } + + std::pair equal_range( const key_type& key ) const { + auto result = const_cast(this)->internal_equal_range(key); + return std::make_pair(const_iterator(result.first), const_iterator(result.second)); + } + + template + typename std::enable_if::value, std::pair>::type equal_range( const K& key ) { + auto result = internal_equal_range(key); + return std::make_pair(iterator(result.first), iterator(result.second)); + } + + template + typename std::enable_if::value, std::pair>::type equal_range( const K& key ) const { + auto result = const_cast(this)->internal_equal_range(key); + return std::make_pair(iterator(result.first), iterator(result.second)); + } + + size_type count( const key_type& key ) const { + return internal_count(key); + } + + template + typename std::enable_if::value, size_type>::type count( const K& key ) const { + return internal_count(key); + } + + bool contains( const key_type& key ) const { + return find(key) != end(); + } + + template + typename std::enable_if::value, bool>::type contains( const K& key ) const { + return find(key) != end(); + } + + // Bucket interface + local_iterator unsafe_begin( size_type n ) { + return local_iterator(first_value_node(get_bucket(n))); + } + + const_local_iterator unsafe_begin( size_type n ) const { + auto bucket_begin = first_value_node(const_cast(this)->get_bucket(n)); + return const_local_iterator(bucket_begin); + } + + const_local_iterator unsafe_cbegin( size_type n ) const { + auto bucket_begin = first_value_node(const_cast(this)->get_bucket(n)); + return const_local_iterator(bucket_begin); + } + + local_iterator unsafe_end( size_type n ) { + size_type bucket_count = my_bucket_count.load(std::memory_order_relaxed); + return n != bucket_count - 1 ? unsafe_begin(get_next_bucket_index(n)) : local_iterator(nullptr); + } + + const_local_iterator unsafe_end( size_type n ) const { + size_type bucket_count = my_bucket_count.load(std::memory_order_relaxed); + return n != bucket_count - 1 ? unsafe_begin(get_next_bucket_index(n)) : const_local_iterator(nullptr); + } + + const_local_iterator unsafe_cend( size_type n ) const { + size_type bucket_count = my_bucket_count.load(std::memory_order_relaxed); + return n != bucket_count - 1 ? unsafe_begin(get_next_bucket_index(n)) : const_local_iterator(nullptr); + } + + size_type unsafe_bucket_count() const { return my_bucket_count.load(std::memory_order_relaxed); } + + size_type unsafe_max_bucket_count() const { + return max_size(); + } + + size_type unsafe_bucket_size( size_type n ) const { + return size_type(std::distance(unsafe_begin(n), unsafe_end(n))); + } + + size_type unsafe_bucket( const key_type& key ) const { + return my_hash_compare(key) % my_bucket_count.load(std::memory_order_relaxed); + } + + // Hash policy + float load_factor() const { + return float(size() / float(my_bucket_count.load(std::memory_order_acquire))); + } + + float max_load_factor() const { return my_max_load_factor; } + + void max_load_factor( float mlf ) { + if (mlf != mlf || mlf < 0) { + tbb::detail::throw_exception(exception_id::invalid_load_factor); + } + my_max_load_factor = mlf; + } // TODO: unsafe? + + void rehash( size_type bucket_count ) { + size_type current_bucket_count = my_bucket_count.load(std::memory_order_acquire); + if (current_bucket_count < bucket_count) { + // TODO: do we need do-while here? + my_bucket_count.compare_exchange_strong(current_bucket_count, round_up_to_power_of_two(bucket_count)); + } + } + + void reserve( size_type elements_count ) { + size_type current_bucket_count = my_bucket_count.load(std::memory_order_acquire); + size_type necessary_bucket_count = current_bucket_count; + + // max_load_factor() is currently unsafe, so we can assume that my_max_load_factor + // would not be changed during the calculation + // TODO: Log2 seems useful here + while (necessary_bucket_count * max_load_factor() < elements_count) { + necessary_bucket_count <<= 1; + } + + while (!my_bucket_count.compare_exchange_strong(current_bucket_count, necessary_bucket_count)) { + if (current_bucket_count >= necessary_bucket_count) + break; + } + } + + // Observers + hasher hash_function() const { return my_hash_compare.hash_function(); } + key_equal key_eq() const { return my_hash_compare.key_eq(); } + + class const_range_type { + private: + const concurrent_unordered_base& my_instance; + node_ptr my_begin_node; // may be node* const + node_ptr my_end_node; + mutable node_ptr my_midpoint_node; + public: + using size_type = typename concurrent_unordered_base::size_type; + using value_type = typename concurrent_unordered_base::value_type; + using reference = typename concurrent_unordered_base::reference; + using difference_type = typename concurrent_unordered_base::difference_type; + using iterator = typename concurrent_unordered_base::const_iterator; + + bool empty() const { return my_begin_node == my_end_node; } + + bool is_divisible() const { + return my_midpoint_node != my_end_node; + } + + size_type grainsize() const { return 1; } + + const_range_type( const_range_type& range, split ) + : my_instance(range.my_instance), + my_begin_node(range.my_midpoint_node), + my_end_node(range.my_end_node) + { + range.my_end_node = my_begin_node; + __TBB_ASSERT(!empty(), "Splitting despite the range is not divisible"); + __TBB_ASSERT(!range.empty(), "Splitting despite the range is not divisible"); + set_midpoint(); + range.set_midpoint(); + } + + iterator begin() const { return iterator(my_instance.first_value_node(my_begin_node)); } + iterator end() const { return iterator(my_instance.first_value_node(my_end_node)); } + + const_range_type( const concurrent_unordered_base& table ) + : my_instance(table), my_begin_node(my_instance.first_value_node(const_cast(&table.my_head))), my_end_node(nullptr) + { + set_midpoint(); + } + private: + void set_midpoint() const { + if (empty()) { + my_midpoint_node = my_end_node; + } else { + sokey_type invalid_key = ~sokey_type(0); + sokey_type begin_key = my_begin_node != nullptr ? my_begin_node->order_key() : invalid_key; + sokey_type end_key = my_end_node != nullptr ? my_end_node->order_key() : invalid_key; + + size_type mid_bucket = reverse_bits(begin_key + (end_key - begin_key) / 2) % + my_instance.my_bucket_count.load(std::memory_order_relaxed); + while( my_instance.my_segments[mid_bucket].load(std::memory_order_relaxed) == nullptr) { + mid_bucket = my_instance.get_parent(mid_bucket); + } + if (reverse_bits(mid_bucket) > begin_key) { + // Found a dummy node between begin and end + my_midpoint_node = my_instance.first_value_node( + my_instance.my_segments[mid_bucket].load(std::memory_order_relaxed)); + } else { + // Didn't find a dummy node between begin and end + my_midpoint_node = my_end_node; + } + } + } + }; // class const_range_type + + class range_type : public const_range_type { + public: + using iterator = typename concurrent_unordered_base::iterator; + using const_range_type::const_range_type; + + iterator begin() const { return iterator(const_range_type::begin().get_node_ptr()); } + iterator end() const { return iterator(const_range_type::end().get_node_ptr()); } + }; // class range_type + + // Parallel iteration + range_type range() { + return range_type(*this); + } + + const_range_type range() const { + return const_range_type(*this); + } +protected: + static constexpr bool allow_multimapping = traits_type::allow_multimapping; + +private: + static constexpr size_type initial_bucket_count = 8; + static constexpr float initial_max_load_factor = 4; // TODO: consider 1? + static constexpr size_type pointers_per_embedded_table = sizeof(size_type) * 8 - 1; + + class unordered_segment_table + : public d1::segment_table, allocator_type, unordered_segment_table, pointers_per_embedded_table> + { + using self_type = unordered_segment_table; + using atomic_node_ptr = std::atomic; + using base_type = d1::segment_table, allocator_type, unordered_segment_table, pointers_per_embedded_table>; + using segment_type = typename base_type::segment_type; + using base_allocator_type = typename base_type::allocator_type; + + using segment_allocator_type = typename allocator_traits_type::template rebind_alloc; + using segment_allocator_traits = tbb::detail::allocator_traits; + public: + // Segment table for unordered containers should not be extended in the wait- free implementation + static constexpr bool allow_table_extending = false; + static constexpr bool is_noexcept_assignment = std::is_nothrow_move_assignable::value && + std::is_nothrow_move_assignable::value && + segment_allocator_traits::is_always_equal::value; + static constexpr bool is_noexcept_swap = tbb::detail::is_nothrow_swappable::value && + tbb::detail::is_nothrow_swappable::value && + segment_allocator_traits::is_always_equal::value; + + // TODO: using base_type::base_type is not compiling on Windows and Intel Compiler - investigate + unordered_segment_table( const base_allocator_type& alloc = base_allocator_type() ) + : base_type(alloc) {} + + unordered_segment_table( const unordered_segment_table& ) = default; + + unordered_segment_table( const unordered_segment_table& other, const base_allocator_type& alloc ) + : base_type(other, alloc) {} + + unordered_segment_table( unordered_segment_table&& ) = default; + + unordered_segment_table( unordered_segment_table&& other, const base_allocator_type& alloc ) + : base_type(std::move(other), alloc) {} + + unordered_segment_table& operator=( const unordered_segment_table& ) = default; + + unordered_segment_table& operator=( unordered_segment_table&& ) = default; + + segment_type create_segment( typename base_type::segment_table_type, typename base_type::segment_index_type segment_index, size_type ) { + segment_allocator_type alloc(this->get_allocator()); + size_type seg_size = this->segment_size(segment_index); + segment_type new_segment = segment_allocator_traits::allocate(alloc, seg_size); + for (size_type i = 0; i != seg_size; ++i) { + segment_allocator_traits::construct(alloc, new_segment + i, nullptr); + } + return new_segment; + } + + segment_type nullify_segment( typename base_type::segment_table_type table, size_type segment_index ) { + segment_type target_segment = table[segment_index].load(std::memory_order_relaxed); + table[segment_index].store(nullptr, std::memory_order_relaxed); + return target_segment; + } + + // deallocate_segment is required by the segment_table base class, but + // in unordered, it is also necessary to call the destructor during deallocation + void deallocate_segment( segment_type address, size_type index ) { + destroy_segment(address, index); + } + + void destroy_segment( segment_type address, size_type index ) { + segment_allocator_type alloc(this->get_allocator()); + for (size_type i = 0; i != this->segment_size(index); ++i) { + segment_allocator_traits::destroy(alloc, address + i); + } + segment_allocator_traits::deallocate(alloc, address, this->segment_size(index)); + } + + + void copy_segment( size_type index, segment_type, segment_type to ) { + if (index == 0) { + // The first element in the first segment is embedded into the table (my_head) + // so the first pointer should not be stored here + // It would be stored during move ctor/assignment operation + to[1].store(nullptr, std::memory_order_relaxed); + } else { + for (size_type i = 0; i != this->segment_size(index); ++i) { + to[i].store(nullptr, std::memory_order_relaxed); + } + } + } + + void move_segment( size_type index, segment_type from, segment_type to ) { + if (index == 0) { + // The first element in the first segment is embedded into the table (my_head) + // so the first pointer should not be stored here + // It would be stored during move ctor/assignment operation + to[1].store(from[1].load(std::memory_order_relaxed), std::memory_order_relaxed); + } else { + for (size_type i = 0; i != this->segment_size(index); ++i) { + to[i].store(from[i].load(std::memory_order_relaxed), std::memory_order_relaxed); + from[i].store(nullptr, std::memory_order_relaxed); + } + } + } + + // allocate_long_table is required by the segment_table base class, but unused for unordered containers + typename base_type::segment_table_type allocate_long_table( const typename base_type::atomic_segment*, size_type ) { + __TBB_ASSERT(false, "This method should never been called"); + // TableType is a pointer + return nullptr; + } + + // destroy_elements is required by the segment_table base class, but unused for unordered containers + // this function call but do nothing + void destroy_elements() {} + }; // struct unordered_segment_table + + void internal_clear() { + // TODO: consider usefulness of two versions of clear() - with dummy nodes deallocation and without it + node_ptr next = my_head.next(); + node_ptr curr = next; + + my_head.set_next(nullptr); + + while (curr != nullptr) { + next = curr->next(); + destroy_node(curr); + curr = next; + } + + my_size.store(0, std::memory_order_relaxed); + my_segments.clear(); + } + + void destroy_node( node_ptr node ) { + if (node->is_dummy()) { + node_allocator_type dummy_node_allocator(my_segments.get_allocator()); + // Destroy the node + node_allocator_traits::destroy(dummy_node_allocator, node); + // Deallocate the memory + node_allocator_traits::deallocate(dummy_node_allocator, node, 1); + } else { + // GCC 11.1 issues a warning here that incorrect destructor might be called for dummy_nodes + #if (__TBB_GCC_VERSION >= 110100 && __TBB_GCC_VERSION < 160000 ) && !__clang__ && !__INTEL_COMPILER + volatile + #endif + value_node_ptr val_node = static_cast(node); + value_node_allocator_type value_node_allocator(my_segments.get_allocator()); + // Destroy the value + value_node_allocator_traits::destroy(value_node_allocator, val_node->storage()); + // Destroy the node + value_node_allocator_traits::destroy(value_node_allocator, val_node); + // Deallocate the memory + value_node_allocator_traits::deallocate(value_node_allocator, val_node, 1); + } + } + + struct internal_insert_return_type { + // If the insertion failed - the remaining_node points to the node, which was failed to insert + // This node can be allocated in process of insertion + value_node_ptr remaining_node; + // If the insertion failed - node_with_equal_key points to the node in the list with the + // key, equivalent to the inserted, otherwise it points to the node, which was inserted. + value_node_ptr node_with_equal_key; + // Insertion status + // NOTE: if it is true - remaining_node should be nullptr + bool inserted; + }; // struct internal_insert_return_type + + // Inserts the value into the split ordered list + template + std::pair internal_insert_value( ValueType&& value ) { + + auto create_value_node = [&value, this]( sokey_type order_key )->value_node_ptr { + return create_node(order_key, std::forward(value)); + }; + + auto insert_result = internal_insert(value, create_value_node); + + if (insert_result.remaining_node != nullptr) { + // If the insertion fails - destroy the node which was failed to insert if it exist + __TBB_ASSERT(!insert_result.inserted, + "remaining_node should be nullptr if the node was successfully inserted"); + destroy_node(insert_result.remaining_node); + } + + return { iterator(insert_result.node_with_equal_key), insert_result.inserted }; + } + + // Inserts the node into the split ordered list + // Creates a node using the specified callback after the place for insertion was found + // Returns internal_insert_return_type object, where: + // - If the insertion succeeded: + // - remaining_node is nullptr + // - node_with_equal_key point to the inserted node + // - inserted is true + // - If the insertion failed: + // - remaining_node points to the node, that was failed to insert if it was created. + // nullptr if the node was not created, because the requested key was already + // presented in the list + // - node_with_equal_key point to the element in the list with the key, equivalent to + // to the requested key + // - inserted is false + template + internal_insert_return_type internal_insert( ValueType&& value, CreateInsertNode create_insert_node ) { + static_assert(std::is_same::type, value_type>::value, + "Incorrect type in internal_insert"); + const key_type& key = traits_type::get_key(value); + sokey_type hash_key = sokey_type(my_hash_compare(key)); + + sokey_type order_key = split_order_key_regular(hash_key); + node_ptr prev = prepare_bucket(hash_key); + __TBB_ASSERT(prev != nullptr, "Invalid head node"); + + auto search_result = search_after(prev, order_key, key); + + if (search_result.second) { + return internal_insert_return_type{ nullptr, search_result.first, false }; + } + + value_node_ptr new_node = create_insert_node(order_key); + node_ptr curr = search_result.first; + + while (!try_insert(prev, new_node, curr)) { + search_result = search_after(prev, order_key, key); + if (search_result.second) { + return internal_insert_return_type{ new_node, search_result.first, false }; + } + curr = search_result.first; + } + + auto sz = my_size.fetch_add(1); + adjust_table_size(sz + 1, my_bucket_count.load(std::memory_order_acquire)); + return internal_insert_return_type{ nullptr, static_cast(new_node), true }; + } + + // Searches the node with the key, equivalent to key with requested order key after the node prev + // Returns the existing node and true if the node is already in the list + // Returns the first node with the order key, greater than requested and false if the node is not presented in the list + std::pair search_after( node_ptr& prev, sokey_type order_key, const key_type& key ) { + // NOTE: static_cast(curr) should be done only after we would ensure + // that the node is not a dummy node + + node_ptr curr = prev->next(); + + while (curr != nullptr && (curr->order_key() < order_key || + (curr->order_key() == order_key && !my_hash_compare(traits_type::get_key(static_cast(curr)->value()), key)))) + { + prev = curr; + curr = curr->next(); + } + + if (curr != nullptr && curr->order_key() == order_key && !allow_multimapping) { + return { static_cast(curr), true }; + } + return { static_cast(curr), false }; + } + + void adjust_table_size( size_type total_elements, size_type current_size ) { + // Grow the table by a factor of 2 if possible and needed + if ( (float(total_elements) / float(current_size)) > my_max_load_factor ) { + // Double the size of the hash only if size hash not changed in between loads + my_bucket_count.compare_exchange_strong(current_size, 2u * current_size); + } + } + + node_ptr insert_dummy_node( node_ptr parent_dummy_node, sokey_type order_key ) { + node_ptr prev_node = parent_dummy_node; + + node_ptr dummy_node = create_dummy_node(order_key); + node_ptr next_node; + + do { + next_node = prev_node->next(); + // Move forward through the list while the order key is less than requested + while (next_node != nullptr && next_node->order_key() < order_key) { + prev_node = next_node; + next_node = next_node->next(); + } + + if (next_node != nullptr && next_node->order_key() == order_key) { + // Another dummy node with the same order key was inserted by another thread + // Destroy the node and exit + destroy_node(dummy_node); + return next_node; + } + } while (!try_insert(prev_node, dummy_node, next_node)); + + return dummy_node; + } + + // Try to insert a node between prev_node and expected next + // If the next is not equal to expected next - return false + static bool try_insert( node_ptr prev_node, node_ptr new_node, node_ptr current_next_node ) { + new_node->set_next(current_next_node); + return prev_node->try_set_next(current_next_node, new_node); + } + + // Returns the bucket, associated with the hash_key + node_ptr prepare_bucket( sokey_type hash_key ) { + size_type bucket = hash_key % my_bucket_count.load(std::memory_order_acquire); + return get_bucket(bucket); + } + + // Initialize the corresponding bucket if it is not initialized + node_ptr get_bucket( size_type bucket_index ) { + if (my_segments[bucket_index].load(std::memory_order_acquire) == nullptr) { + init_bucket(bucket_index); + } + return my_segments[bucket_index].load(std::memory_order_acquire); + } + + void init_bucket( size_type bucket ) { + if (bucket == 0) { + // Atomicaly store the first bucket into my_head + node_ptr disabled = nullptr; + my_segments[0].compare_exchange_strong(disabled, &my_head); + return; + } + + size_type parent_bucket = get_parent(bucket); + + while (my_segments[parent_bucket].load(std::memory_order_acquire) == nullptr) { + // Initialize all of the parent buckets + init_bucket(parent_bucket); + } + + __TBB_ASSERT(my_segments[parent_bucket].load(std::memory_order_acquire) != nullptr, "Parent bucket should be initialized"); + node_ptr parent = my_segments[parent_bucket].load(std::memory_order_acquire); + + // Insert dummy node into the list + node_ptr dummy_node = insert_dummy_node(parent, split_order_key_dummy(bucket)); + // TODO: consider returning pair to avoid store operation if the bucket was stored by an other thread + // or move store to insert_dummy_node + // Add dummy_node into the segment table + my_segments[bucket].store(dummy_node, std::memory_order_release); + } + + node_ptr create_dummy_node( sokey_type order_key ) { + node_allocator_type dummy_node_allocator(my_segments.get_allocator()); + node_ptr dummy_node = node_allocator_traits::allocate(dummy_node_allocator, 1); + node_allocator_traits::construct(dummy_node_allocator, dummy_node, order_key); + return dummy_node; + } + + template + value_node_ptr create_node( sokey_type order_key, Args&&... args ) { + value_node_allocator_type value_node_allocator(my_segments.get_allocator()); + // Allocate memory for the value_node + value_node_ptr new_node = value_node_allocator_traits::allocate(value_node_allocator, 1); + // Construct the node + value_node_allocator_traits::construct(value_node_allocator, new_node, order_key); + + // try_call API is not convenient here due to broken + // variadic capture on GCC 4.8.5 + auto value_guard = make_raii_guard([&] { + value_node_allocator_traits::destroy(value_node_allocator, new_node); + value_node_allocator_traits::deallocate(value_node_allocator, new_node, 1); + }); + + // Construct the value in the node + value_node_allocator_traits::construct(value_node_allocator, new_node->storage(), std::forward(args)...); + value_guard.dismiss(); + return new_node; + } + + value_node_ptr first_value_node( node_ptr first_node ) const { + while (first_node != nullptr && first_node->is_dummy()) { + first_node = first_node->next(); + } + return static_cast(first_node); + } + + // Unsafe method, which removes the node from the list and returns the next node + node_ptr internal_erase( value_node_ptr node_to_erase ) { + __TBB_ASSERT(node_to_erase != nullptr, "Invalid iterator for erase"); + node_ptr next_node = node_to_erase->next(); + internal_extract(node_to_erase); + destroy_node(node_to_erase); + return next_node; + } + + template + size_type internal_erase_by_key( const K& key ) { + // TODO: consider reimplementation without equal_range - it is not effective to perform lookup over a bucket + // for each unsafe_erase call + auto eq_range = equal_range(key); + size_type erased_count = 0; + + for (auto it = eq_range.first; it != eq_range.second;) { + it = unsafe_erase(it); + ++erased_count; + } + return erased_count; + } + + // Unsafe method, which extracts the node from the list + void internal_extract( value_node_ptr node_to_extract ) { + const key_type& key = traits_type::get_key(node_to_extract->value()); + sokey_type hash_key = sokey_type(my_hash_compare(key)); + + node_ptr prev_node = prepare_bucket(hash_key); + + for (node_ptr node = prev_node->next(); node != nullptr; prev_node = node, node = node->next()) { + if (node == node_to_extract) { + unlink_node(prev_node, node, node_to_extract->next()); + my_size.store(my_size.load(std::memory_order_relaxed) - 1, std::memory_order_relaxed); + return; + } + __TBB_ASSERT(node->order_key() <= node_to_extract->order_key(), + "node, which is going to be extracted should be presented in the list"); + } + } + +protected: + template + void internal_merge( SourceType&& source ) { + static_assert(std::is_same::type::node_type>::value, + "Incompatible containers cannot be merged"); + + for (node_ptr source_prev = &source.my_head; source_prev->next() != nullptr;) { + if (!source_prev->next()->is_dummy()) { + value_node_ptr curr = static_cast(source_prev->next()); + // If the multimapping is allowed, or the key is not presented + // in the *this container - extract the node from the list + if (allow_multimapping || !contains(traits_type::get_key(curr->value()))) { + node_ptr next_node = curr->next(); + source.unlink_node(source_prev, curr, next_node); + + // Remember the old order key + sokey_type old_order_key = curr->order_key(); + + // Node handle with curr cannot be used directly in insert call, because + // the destructor of node_type will destroy curr + node_type curr_node = d1::node_handle_accessor::construct(curr); + + // If the insertion fails - return ownership of the node to the source + if (!insert(std::move(curr_node)).second) { + __TBB_ASSERT(!allow_multimapping, "Insertion should succeed for multicontainer"); + __TBB_ASSERT(source_prev->next() == next_node, + "Concurrent operations with the source container in merge are prohibited"); + + // Initialize the node with the old order key, because the order key + // can change during the insertion + curr->init(old_order_key); + __TBB_ASSERT(old_order_key >= source_prev->order_key() && + (next_node == nullptr || old_order_key <= next_node->order_key()), + "Wrong nodes order in the source container"); + // Merge is unsafe for source container, so the insertion back can be done without compare_exchange + curr->set_next(next_node); + source_prev->set_next(curr); + source_prev = curr; + d1::node_handle_accessor::deactivate(curr_node); + } else { + source.my_size.fetch_sub(1, std::memory_order_relaxed); + } + } else { + source_prev = curr; + } + } else { + source_prev = source_prev->next(); + } + } + } + +private: + // Unsafe method, which unlinks the node between prev and next + void unlink_node( node_ptr prev_node, node_ptr node_to_unlink, node_ptr next_node ) { + __TBB_ASSERT(prev_node->next() == node_to_unlink && + node_to_unlink->next() == next_node, + "erasing and extracting nodes from the containers are unsafe in concurrent mode"); + prev_node->set_next(next_node); + node_to_unlink->set_next(nullptr); + } + + template + value_node_ptr internal_find( const K& key ) { + sokey_type hash_key = sokey_type(my_hash_compare(key)); + sokey_type order_key = split_order_key_regular(hash_key); + + node_ptr curr = prepare_bucket(hash_key); + + while (curr != nullptr) { + if (curr->order_key() > order_key) { + // If the order key is greater than the requested order key, + // the element is not in the hash table + return nullptr; + } else if (curr->order_key() == order_key && + my_hash_compare(traits_type::get_key(static_cast(curr)->value()), key)) { + // The fact that order keys match does not mean that the element is found. + // Key function comparison has to be performed to check whether this is the + // right element. If not, keep searching while order key is the same. + return static_cast(curr); + } + curr = curr->next(); + } + + return nullptr; + } + + template + std::pair internal_equal_range( const K& key ) { + sokey_type hash_key = sokey_type(my_hash_compare(key)); + sokey_type order_key = split_order_key_regular(hash_key); + + node_ptr curr = prepare_bucket(hash_key); + + while (curr != nullptr) { + if (curr->order_key() > order_key) { + // If the order key is greater than the requested order key, + // the element is not in the hash table + return std::make_pair(nullptr, nullptr); + } else if (curr->order_key() == order_key && + my_hash_compare(traits_type::get_key(static_cast(curr)->value()), key)) { + value_node_ptr first = static_cast(curr); + node_ptr last = first; + do { + last = last->next(); + } while (allow_multimapping && last != nullptr && !last->is_dummy() && + my_hash_compare(traits_type::get_key(static_cast(last)->value()), key)); + return std::make_pair(first, first_value_node(last)); + } + curr = curr->next(); + } + return {nullptr, nullptr}; + } + + template + size_type internal_count( const K& key ) const { + if (allow_multimapping) { + // TODO: consider reimplementing the internal_equal_range with elements counting to avoid std::distance + auto eq_range = equal_range(key); + return std::distance(eq_range.first, eq_range.second); + } else { + return contains(key) ? 1 : 0; + } + } + + void internal_copy( const concurrent_unordered_base& other ) { + node_ptr last_node = &my_head; + my_segments[0].store(&my_head, std::memory_order_relaxed); + + for (node_ptr node = other.my_head.next(); node != nullptr; node = node->next()) { + node_ptr new_node; + if (!node->is_dummy()) { + // The node in the right table contains a value + new_node = create_node(node->order_key(), static_cast(node)->value()); + } else { + // The node in the right table is a dummy node + new_node = create_dummy_node(node->order_key()); + my_segments[reverse_bits(node->order_key())].store(new_node, std::memory_order_relaxed); + } + + last_node->set_next(new_node); + last_node = new_node; + } + } + + void internal_move( concurrent_unordered_base&& other ) { + node_ptr last_node = &my_head; + my_segments[0].store(&my_head, std::memory_order_relaxed); + + for (node_ptr node = other.my_head.next(); node != nullptr; node = node->next()) { + node_ptr new_node; + if (!node->is_dummy()) { + // The node in the right table contains a value + new_node = create_node(node->order_key(), std::move(static_cast(node)->value())); + } else { + // TODO: do we need to destroy a dummy node in the right container? + // The node in the right table is a dummy_node + new_node = create_dummy_node(node->order_key()); + my_segments[reverse_bits(node->order_key())].store(new_node, std::memory_order_relaxed); + } + + last_node->set_next(new_node); + last_node = new_node; + } + } + + void move_content( concurrent_unordered_base&& other ) { + // NOTE: allocators should be equal + my_head.set_next(other.my_head.next()); + other.my_head.set_next(nullptr); + my_segments[0].store(&my_head, std::memory_order_relaxed); + + other.my_bucket_count.store(initial_bucket_count, std::memory_order_relaxed); + other.my_max_load_factor = initial_max_load_factor; + other.my_size.store(0, std::memory_order_relaxed); + } + + void internal_move_construct_with_allocator( concurrent_unordered_base&& other, const allocator_type&, + /*is_always_equal = */std::true_type ) { + // Allocators are always equal - no need to compare for equality + move_content(std::move(other)); + } + + void internal_move_construct_with_allocator( concurrent_unordered_base&& other, const allocator_type& alloc, + /*is_always_equal = */std::false_type ) { + // Allocators are not always equal + if (alloc == other.my_segments.get_allocator()) { + move_content(std::move(other)); + } else { + try_call( [&] { + internal_move(std::move(other)); + } ).on_exception( [&] { + clear(); + }); + } + } + + // Move assigns the hash table to other is any instances of allocator_type are always equal + // or propagate_on_container_move_assignment is true + void internal_move_assign( concurrent_unordered_base&& other, /*is_always_equal || POCMA = */std::true_type ) { + move_content(std::move(other)); + } + + // Move assigns the hash table to other is any instances of allocator_type are not always equal + // and propagate_on_container_move_assignment is false + void internal_move_assign( concurrent_unordered_base&& other, /*is_always_equal || POCMA = */std::false_type ) { + if (my_segments.get_allocator() == other.my_segments.get_allocator()) { + move_content(std::move(other)); + } else { + // TODO: guards for exceptions + internal_move(std::move(other)); + } + } + + void internal_swap( concurrent_unordered_base& other, /*is_always_equal || POCS = */std::true_type ) { + internal_swap_fields(other); + } + + void internal_swap( concurrent_unordered_base& other, /*is_always_equal || POCS = */std::false_type ) { + __TBB_ASSERT(my_segments.get_allocator() == other.my_segments.get_allocator(), + "Swapping with unequal allocators is not allowed"); + internal_swap_fields(other); + } + + void internal_swap_fields( concurrent_unordered_base& other ) { + node_ptr first_node = my_head.next(); + my_head.set_next(other.my_head.next()); + other.my_head.set_next(first_node); + + size_type current_size = my_size.load(std::memory_order_relaxed); + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.my_size.store(current_size, std::memory_order_relaxed); + + size_type bucket_count = my_bucket_count.load(std::memory_order_relaxed); + my_bucket_count.store(other.my_bucket_count.load(std::memory_order_relaxed), std::memory_order_relaxed); + other.my_bucket_count.store(bucket_count, std::memory_order_relaxed); + + using std::swap; + swap(my_max_load_factor, other.my_max_load_factor); + swap(my_hash_compare, other.my_hash_compare); + my_segments.swap(other.my_segments); + + // swap() method from segment table swaps all of the segments including the first segment + // We should restore it to my_head. Without it the first segment of the container will point + // to other.my_head. + my_segments[0].store(&my_head, std::memory_order_relaxed); + other.my_segments[0].store(&other.my_head, std::memory_order_relaxed); + } + + // A regular order key has its original hash value reversed and the last bit set + static constexpr sokey_type split_order_key_regular( sokey_type hash ) { + return reverse_bits(hash) | 0x1; + } + + // A dummy order key has its original hash value reversed and the last bit unset + static constexpr sokey_type split_order_key_dummy( sokey_type hash ) { + return reverse_bits(hash) & ~sokey_type(0x1); + } + + size_type get_parent( size_type bucket ) const { + // Unset bucket's most significant turned-on bit + __TBB_ASSERT(bucket != 0, "Unable to get_parent of the bucket 0"); + size_type msb = tbb::detail::log2(bucket); + return bucket & ~(size_type(1) << msb); + } + + size_type get_next_bucket_index( size_type bucket ) const { + size_type bits = tbb::detail::log2(my_bucket_count.load(std::memory_order_relaxed)); + size_type reversed_next = reverse_n_bits(bucket, bits) + 1; + return reverse_n_bits(reversed_next, bits); + } + + std::atomic my_size; + std::atomic my_bucket_count; + float my_max_load_factor; + hash_compare_type my_hash_compare; + + list_node_type my_head; // Head node for split ordered list + unordered_segment_table my_segments; // Segment table of pointers to nodes + + template + friend class solist_iterator; + + template + friend class concurrent_unordered_base; +}; // class concurrent_unordered_base + +template +bool operator==( const concurrent_unordered_base& lhs, + const concurrent_unordered_base& rhs ) { + if (&lhs == &rhs) { return true; } + if (lhs.size() != rhs.size()) { return false; } + +#if _MSC_VER + // Passing "unchecked" iterators to std::permutation with 3 parameters + // causes compiler warnings. + // The workaround is to use overload with 4 parameters, which is + // available since C++14 - minimally supported version on MSVC + return std::is_permutation(lhs.begin(), lhs.end(), rhs.begin(), rhs.end()); +#else + return std::is_permutation(lhs.begin(), lhs.end(), rhs.begin()); +#endif +} + +#if !__TBB_CPP20_COMPARISONS_PRESENT +template +bool operator!=( const concurrent_unordered_base& lhs, + const concurrent_unordered_base& rhs ) { + return !(lhs == rhs); +} +#endif + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(pop) // warning 4127 is back +#endif + +} // namespace d2 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__concurrent_unordered_base_H diff --git a/src/3rd party/oneapi/tbb/detail/_config.h b/src/3rd party/oneapi/tbb/detail/_config.h new file mode 100644 index 0000000000..1cf8ac3538 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_config.h @@ -0,0 +1,581 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + Copyright (c) 2025 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__config_H +#define __TBB_detail__config_H + +/** This header is supposed to contain macro definitions only. + The macros defined here are intended to control such aspects of TBB build as + - presence of compiler features + - compilation modes + - feature sets + - known compiler/platform issues +**/ + +/* Check which standard library we use. */ +#include + +#ifdef __has_include +#if __has_include() +#include +#endif +#endif + +#include "_export.h" + +#if _MSC_VER + #define __TBB_EXPORTED_FUNC __cdecl + #define __TBB_EXPORTED_METHOD __thiscall +#else + #define __TBB_EXPORTED_FUNC + #define __TBB_EXPORTED_METHOD +#endif + +#if defined(_MSVC_LANG) + #define __TBB_LANG _MSVC_LANG +#else + #define __TBB_LANG __cplusplus +#endif // _MSVC_LANG + +#define __TBB_CPP14_PRESENT (__TBB_LANG >= 201402L) +#define __TBB_CPP17_PRESENT (__TBB_LANG >= 201703L) +#define __TBB_CPP20_PRESENT (__TBB_LANG >= 202002L) + +#if __TBB_CPP17_PRESENT + #define __TBB_GLOBAL_VAR inline +#else + #define __TBB_GLOBAL_VAR static +#endif + +#if __INTEL_COMPILER || _MSC_VER + #define __TBB_NOINLINE(decl) __declspec(noinline) decl +#elif __GNUC__ + #define __TBB_NOINLINE(decl) decl __attribute__ ((noinline)) +#else + #define __TBB_NOINLINE(decl) decl +#endif + +#define __TBB_STRING_AUX(x) #x +#define __TBB_STRING(x) __TBB_STRING_AUX(x) + +// Note that when ICC or Clang is in use, __TBB_GCC_VERSION might not fully match +// the actual GCC version on the system. +#define __TBB_GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) + +/* Check which standard library we use. */ + +// Prior to GCC 7, GNU libstdc++ did not have a convenient version macro. +// Therefore we use different ways to detect its version. +#ifdef TBB_USE_GLIBCXX_VERSION + // The version is explicitly specified in our public TBB_USE_GLIBCXX_VERSION macro. + // Its format should match the __TBB_GCC_VERSION above, e.g. 70301 for libstdc++ coming with GCC 7.3.1. + #define __TBB_GLIBCXX_VERSION TBB_USE_GLIBCXX_VERSION +#elif _GLIBCXX_RELEASE && _GLIBCXX_RELEASE != __GNUC__ + // Reported versions of GCC and libstdc++ do not match; trust the latter + #define __TBB_GLIBCXX_VERSION (_GLIBCXX_RELEASE*10000) +#elif __GLIBCPP__ || __GLIBCXX__ + // The version macro is not defined or matches the GCC version; use __TBB_GCC_VERSION + #define __TBB_GLIBCXX_VERSION __TBB_GCC_VERSION +#endif + +#if __clang__ + // according to clang documentation, version can be vendor specific + #define __TBB_CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) +#endif + +/** Macro helpers **/ + +#define __TBB_CONCAT_AUX(A,B) A##B +// The additional level of indirection is needed to expand macros A and B (not to get the AB macro). +// See [cpp.subst] and [cpp.concat] for more details. +#define __TBB_CONCAT(A,B) __TBB_CONCAT_AUX(A,B) +// The IGNORED argument and comma are needed to always have 2 arguments (even when A is empty). +#define __TBB_IS_MACRO_EMPTY(A,IGNORED) __TBB_CONCAT_AUX(__TBB_MACRO_EMPTY,A) +#define __TBB_MACRO_EMPTY 1 + +#if _M_X64 || _M_ARM64 + #define __TBB_W(name) name##64 +#else + #define __TBB_W(name) name +#endif + +/** User controlled TBB features & modes **/ + +#ifndef TBB_USE_DEBUG + /* + There are four cases that are supported: + 1. "_DEBUG is undefined" means "no debug"; + 2. "_DEBUG defined to something that is evaluated to 0" (including "garbage", as per [cpp.cond]) means "no debug"; + 3. "_DEBUG defined to something that is evaluated to a non-zero value" means "debug"; + 4. "_DEBUG defined to nothing (empty)" means "debug". + */ + #ifdef _DEBUG + // Check if _DEBUG is empty. + #define __TBB_IS__DEBUG_EMPTY (__TBB_IS_MACRO_EMPTY(_DEBUG,IGNORED)==__TBB_MACRO_EMPTY) + #if __TBB_IS__DEBUG_EMPTY + #define TBB_USE_DEBUG 1 + #else + #define TBB_USE_DEBUG _DEBUG + #endif // __TBB_IS__DEBUG_EMPTY + #else + #define TBB_USE_DEBUG 0 + #endif // _DEBUG +#endif // TBB_USE_DEBUG + +#ifndef TBB_USE_ASSERT + #define TBB_USE_ASSERT TBB_USE_DEBUG +#endif // TBB_USE_ASSERT + +#ifndef TBB_USE_PROFILING_TOOLS +#if TBB_USE_DEBUG + #define TBB_USE_PROFILING_TOOLS 2 +#else // TBB_USE_DEBUG + #define TBB_USE_PROFILING_TOOLS 0 +#endif // TBB_USE_DEBUG +#endif // TBB_USE_PROFILING_TOOLS + +// Exceptions support cases +#if !(__EXCEPTIONS || defined(_CPPUNWIND) || __SUNPRO_CC) + #if TBB_USE_EXCEPTIONS + #error Compilation settings do not support exception handling. Please do not set TBB_USE_EXCEPTIONS macro or set it to 0. + #elif !defined(TBB_USE_EXCEPTIONS) + #define TBB_USE_EXCEPTIONS 0 + #endif +#elif !defined(TBB_USE_EXCEPTIONS) + #define TBB_USE_EXCEPTIONS 1 +#endif + +/** Preprocessor symbols to determine HW architecture **/ + +#if _WIN32 || _WIN64 + #if defined(_M_X64) || defined(__x86_64__) // the latter for MinGW support + #define __TBB_x86_64 1 + #elif defined(_M_IA64) + #define __TBB_ipf 1 + #elif defined(_M_IX86) || defined(__i386__) // the latter for MinGW support + #define __TBB_x86_32 1 + #else + #define __TBB_generic_arch 1 + #endif +#else /* Assume generic Unix */ + #if __x86_64__ + #define __TBB_x86_64 1 + #elif __ia64__ + #define __TBB_ipf 1 + #elif __i386__||__i386 // __i386 is for Sun OS + #define __TBB_x86_32 1 + #else + #define __TBB_generic_arch 1 + #endif +#endif + +/** Windows API or POSIX API **/ + +#if _WIN32 || _WIN64 + #define __TBB_USE_WINAPI 1 +#else + #define __TBB_USE_POSIX 1 +#endif + +/** Internal TBB features & modes **/ + +/** __TBB_DYNAMIC_LOAD_ENABLED describes the system possibility to load shared libraries at run time **/ +#ifndef __TBB_DYNAMIC_LOAD_ENABLED + #define __TBB_DYNAMIC_LOAD_ENABLED (!__EMSCRIPTEN__) +#endif + +/** __TBB_WIN8UI_SUPPORT enables support of Windows* Store Apps and limit a possibility to load + shared libraries at run time only from application container **/ +#if defined(WINAPI_FAMILY) && WINAPI_FAMILY == WINAPI_FAMILY_APP + #define __TBB_WIN8UI_SUPPORT 1 +#else + #define __TBB_WIN8UI_SUPPORT 0 +#endif + +/** __TBB_WEAK_SYMBOLS_PRESENT denotes that the system supports the weak symbol mechanism **/ +#ifndef __TBB_WEAK_SYMBOLS_PRESENT + #define __TBB_WEAK_SYMBOLS_PRESENT ( !__EMSCRIPTEN__ && !_WIN32 && !__APPLE__ && !__sun && (__TBB_GCC_VERSION >= 40000 || __INTEL_COMPILER ) ) +#endif + +/** Presence of compiler features **/ + +#if __clang__ && !__INTEL_COMPILER + #define __TBB_USE_OPTIONAL_RTTI __has_feature(cxx_rtti) +#elif defined(_CPPRTTI) + #define __TBB_USE_OPTIONAL_RTTI 1 +#else + #define __TBB_USE_OPTIONAL_RTTI (__GXX_RTTI || __RTTI || __INTEL_RTTI__) +#endif + +/** Address sanitizer detection **/ +#ifdef __SANITIZE_ADDRESS__ + #define __TBB_USE_ADDRESS_SANITIZER 1 +#elif defined(__has_feature) +#if __has_feature(address_sanitizer) + #define __TBB_USE_ADDRESS_SANITIZER 1 +#endif +#endif + +/** Library features presence macros **/ + +#define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (__TBB_LANG >= 201402L) +#define __TBB_CPP17_INVOKE_PRESENT (__TBB_LANG >= 201703L) + +// TODO: Remove the condition(__INTEL_COMPILER > 2021) from the __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +// macro when this feature start working correctly on this compiler. +#if __INTEL_COMPILER && (!_MSC_VER || __INTEL_CXX11_MOVE__) + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__TBB_LANG >= 201402L) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__INTEL_COMPILER > 2021 && __TBB_LANG >= 201703L) +#elif __clang__ + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__has_feature(cxx_variable_templates)) + #ifdef __cpp_deduction_guides + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__cpp_deduction_guides >= 201611L) + #else + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT 0 + #endif +#elif __GNUC__ + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__TBB_LANG >= 201402L && __TBB_GCC_VERSION >= 50000) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__cpp_deduction_guides >= 201606L) +#elif _MSC_VER + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (_MSC_FULL_VER >= 190023918 && (!__INTEL_COMPILER || __INTEL_COMPILER >= 1700)) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (_MSC_VER >= 1914 && __TBB_LANG >= 201703L && (!__INTEL_COMPILER || __INTEL_COMPILER > 2021)) +#else + #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__TBB_LANG >= 201402L) + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__TBB_LANG >= 201703L) +#endif + +// GCC4.8 on RHEL7 does not support std::get_new_handler +#define __TBB_CPP11_GET_NEW_HANDLER_PRESENT (_MSC_VER >= 1900 || __TBB_GLIBCXX_VERSION >= 40900 && __GXX_EXPERIMENTAL_CXX0X__ || _LIBCPP_VERSION) +// GCC4.8 on RHEL7 does not support std::is_trivially_copyable +#define __TBB_CPP11_TYPE_PROPERTIES_PRESENT (_LIBCPP_VERSION || _MSC_VER >= 1700 || (__TBB_GLIBCXX_VERSION >= 50000 && __GXX_EXPERIMENTAL_CXX0X__)) + +#define __TBB_CPP17_MEMORY_RESOURCE_PRESENT (_MSC_VER >= 1913 && (__TBB_LANG > 201402L) || \ + __TBB_GLIBCXX_VERSION >= 90000 && __TBB_LANG >= 201703L) +#define __TBB_CPP17_HW_INTERFERENCE_SIZE_PRESENT (_MSC_VER >= 1911) +#define __TBB_CPP17_LOGICAL_OPERATIONS_PRESENT (__TBB_LANG >= 201703L) +#define __TBB_CPP17_ALLOCATOR_IS_ALWAYS_EQUAL_PRESENT (__TBB_LANG >= 201703L) +#define __TBB_CPP17_IS_SWAPPABLE_PRESENT (__TBB_LANG >= 201703L) + +// TODO: fix concepts on Clang or define the broken versions +#if !(__clang__) && defined(__cpp_concepts) && defined(__cpp_lib_concepts) + #define __TBB_CPP20_CONCEPTS_PRESENT ((__cpp_concepts >= 201907L) && (__cpp_lib_concepts >= 202002L)) +#else + #define __TBB_CPP20_CONCEPTS_PRESENT 0 +#endif + +#if defined(__cpp_impl_three_way_comparison) && defined(__cpp_lib_three_way_comparison) + #define __TBB_CPP20_COMPARISONS_PRESENT ((__cpp_impl_three_way_comparison >= 201907L) && (__cpp_lib_three_way_comparison >= 201907L)) +#else + #define __TBB_CPP20_COMPARISONS_PRESENT 0 +#endif + +#define __TBB_RESUMABLE_TASKS (!__TBB_WIN8UI_SUPPORT && !__ANDROID__ && !__QNXNTO__ && (!__linux__ || __GLIBC__)) + +/* This macro marks incomplete code or comments describing ideas which are considered for the future. + * See also for plain comment with TODO and FIXME marks for small improvement opportunities. + */ +#define __TBB_TODO 0 + +/* Check which standard library we use. */ +/* __TBB_SYMBOL is defined only while processing exported symbols list where C++ is not allowed. */ +#if !defined(__TBB_SYMBOL) && !__TBB_CONFIG_PREPROC_ONLY + #include +#endif + +/** Target OS is either iOS* or iOS* simulator **/ +#if __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ + #define __TBB_IOS 1 +#endif + +#if __APPLE__ + #if __INTEL_COMPILER && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ > 1099 \ + && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101000 + // ICC does not correctly set the macro if -mmacosx-min-version is not specified + #define __TBB_MACOS_TARGET_VERSION (100000 + 10*(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ - 1000)) + #else + #define __TBB_MACOS_TARGET_VERSION __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ + #endif +#endif + +#if defined(__GNUC__) && !defined(__INTEL_COMPILER) + #define __TBB_GCC_WARNING_IGNORED_ATTRIBUTES_PRESENT (__TBB_GCC_VERSION >= 60100) +#endif + +#if __GNUC__ && !__INTEL_COMPILER && !__clang__ + #define __TBB_GCC_PARAMETER_PACK_IN_LAMBDAS_BROKEN (__TBB_GCC_VERSION <= 40805) +#endif + +#define __TBB_CPP17_FALLTHROUGH_PRESENT (__TBB_LANG >= 201703L) +#define __TBB_CPP17_NODISCARD_PRESENT (__TBB_LANG >= 201703L) +#define __TBB_FALLTHROUGH_PRESENT (__TBB_GCC_VERSION >= 70000 && !__INTEL_COMPILER) + +#if __TBB_CPP17_FALLTHROUGH_PRESENT + #define __TBB_fallthrough [[fallthrough]] +#elif __TBB_FALLTHROUGH_PRESENT + #define __TBB_fallthrough __attribute__ ((fallthrough)) +#else + #define __TBB_fallthrough +#endif + +#if __TBB_CPP17_NODISCARD_PRESENT + #define __TBB_nodiscard [[nodiscard]] +#elif __clang__ || __GNUC__ + #define __TBB_nodiscard __attribute__((warn_unused_result)) +#else + #define __TBB_nodiscard +#endif + +#define __TBB_CPP17_UNCAUGHT_EXCEPTIONS_PRESENT (_MSC_VER >= 1900 || __GLIBCXX__ && __cpp_lib_uncaught_exceptions \ + || _LIBCPP_VERSION >= 3700 && (!__TBB_MACOS_TARGET_VERSION || __TBB_MACOS_TARGET_VERSION >= 101200)) + +#define __TBB_TSX_INTRINSICS_PRESENT (__RTM__ || __INTEL_COMPILER || (_MSC_VER>=1700 && (__TBB_x86_64 || __TBB_x86_32))) + +#define __TBB_WAITPKG_INTRINSICS_PRESENT ((__INTEL_COMPILER >= 1900 || (__TBB_GCC_VERSION >= 110000 && (__APPLE__ || __TBB_GNU_ASM_VERSION >= 2032)) || __TBB_CLANG_VERSION >= 120000) \ + && (_WIN32 || _WIN64 || __unix__ || __APPLE__) && (__TBB_x86_32 || __TBB_x86_64) && !__ANDROID__) + +/** Internal TBB features & modes **/ + +/** __TBB_SOURCE_DIRECTLY_INCLUDED is a mode used in whitebox testing when + it's necessary to test internal functions not exported from TBB DLLs +**/ +#if (_WIN32||_WIN64) && (__TBB_SOURCE_DIRECTLY_INCLUDED || TBB_USE_PREVIEW_BINARY) + #define __TBB_NO_IMPLICIT_LINKAGE 1 + #define __TBBMALLOC_NO_IMPLICIT_LINKAGE 1 +#endif + +#if (__TBB_BUILD || __TBBMALLOC_BUILD || __TBBMALLOCPROXY_BUILD || __TBBBIND_BUILD) && !defined(__TBB_NO_IMPLICIT_LINKAGE) + #define __TBB_NO_IMPLICIT_LINKAGE 1 +#endif + +#if _MSC_VER + #if !__TBB_NO_IMPLICIT_LINKAGE + #ifdef _DEBUG + #pragma comment(lib, "tbb12_debug.lib") + #else + #pragma comment(lib, "tbb12.lib") + #endif + #endif +#endif + +#ifndef __TBB_SCHEDULER_OBSERVER + #define __TBB_SCHEDULER_OBSERVER 1 +#endif /* __TBB_SCHEDULER_OBSERVER */ + +#ifndef __TBB_FP_CONTEXT + #define __TBB_FP_CONTEXT 1 +#endif /* __TBB_FP_CONTEXT */ + +#define __TBB_RECYCLE_TO_ENQUEUE __TBB_BUILD // keep non-official + +#ifndef __TBB_ARENA_OBSERVER + #define __TBB_ARENA_OBSERVER __TBB_SCHEDULER_OBSERVER +#endif /* __TBB_ARENA_OBSERVER */ + +// Thread pinning is not available on macOS* +#define __TBB_CPUBIND_PRESENT (!__APPLE__) + +#ifndef __TBB_ENQUEUE_ENFORCED_CONCURRENCY + #define __TBB_ENQUEUE_ENFORCED_CONCURRENCY 1 +#endif + +#if !defined(__TBB_SURVIVE_THREAD_SWITCH) && \ + (_WIN32 || _WIN64 || __APPLE__ || (defined(__unix__) && !__ANDROID__)) + #define __TBB_SURVIVE_THREAD_SWITCH 1 +#endif /* __TBB_SURVIVE_THREAD_SWITCH */ + +#ifndef TBB_PREVIEW_FLOW_GRAPH_FEATURES + #define TBB_PREVIEW_FLOW_GRAPH_FEATURES __TBB_CPF_BUILD +#endif + +#ifndef __TBB_DEFAULT_PARTITIONER + #define __TBB_DEFAULT_PARTITIONER tbb::auto_partitioner +#endif + +#ifndef __TBB_FLOW_TRACE_CODEPTR + #define __TBB_FLOW_TRACE_CODEPTR __TBB_CPF_BUILD +#endif + +// Intel(R) C++ Compiler starts analyzing usages of the deprecated content at the template +// instantiation site, which is too late for suppression of the corresponding messages for internal +// stuff. +#if !defined(__INTEL_COMPILER) && (!defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) || (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0)) + #if (__TBB_LANG >= 201402L && (!defined(_MSC_VER) || _MSC_VER >= 1920)) + #define __TBB_DEPRECATED [[deprecated]] + #define __TBB_DEPRECATED_MSG(msg) [[deprecated(msg)]] + #elif _MSC_VER + #define __TBB_DEPRECATED __declspec(deprecated) + #define __TBB_DEPRECATED_MSG(msg) __declspec(deprecated(msg)) + #elif (__GNUC__ && __TBB_GCC_VERSION >= 40805) || __clang__ + #define __TBB_DEPRECATED __attribute__((deprecated)) + #define __TBB_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) + #endif +#endif // !defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) || (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0) + +#if !defined(__TBB_DEPRECATED) + #define __TBB_DEPRECATED + #define __TBB_DEPRECATED_MSG(msg) +#elif !defined(__TBB_SUPPRESS_INTERNAL_DEPRECATED_MESSAGES) + // Suppress deprecated messages from self + #define __TBB_SUPPRESS_INTERNAL_DEPRECATED_MESSAGES 1 +#endif + +#if defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) && (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0) + #define __TBB_DEPRECATED_VERBOSE __TBB_DEPRECATED + #define __TBB_DEPRECATED_VERBOSE_MSG(msg) __TBB_DEPRECATED_MSG(msg) +#else + #define __TBB_DEPRECATED_VERBOSE + #define __TBB_DEPRECATED_VERBOSE_MSG(msg) +#endif // (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0) + +#if (!defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) || (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0)) && !(__TBB_LANG >= 201103L || _MSC_VER >= 1900) + #pragma message("TBB Warning: Support for C++98/03 is deprecated. Please use the compiler that supports C++11 features at least.") +#endif + +#ifdef _VARIADIC_MAX + #define __TBB_VARIADIC_MAX _VARIADIC_MAX +#else + #if _MSC_VER == 1700 + #define __TBB_VARIADIC_MAX 5 // VS11 setting, issue resolved in VS12 + #elif _MSC_VER == 1600 + #define __TBB_VARIADIC_MAX 10 // VS10 setting + #else + #define __TBB_VARIADIC_MAX 15 + #endif +#endif + +#if __SANITIZE_THREAD__ + #define __TBB_USE_THREAD_SANITIZER 1 +#elif defined(__has_feature) +#if __has_feature(thread_sanitizer) + #define __TBB_USE_THREAD_SANITIZER 1 +#endif +#endif + +#ifndef __TBB_USE_SANITIZERS +#define __TBB_USE_SANITIZERS (__TBB_USE_THREAD_SANITIZER || __TBB_USE_ADDRESS_SANITIZER) +#endif + +#ifndef __TBB_RESUMABLE_TASKS_USE_THREADS +#define __TBB_RESUMABLE_TASKS_USE_THREADS __TBB_USE_SANITIZERS +#endif + +#ifndef __TBB_USE_CONSTRAINTS +#define __TBB_USE_CONSTRAINTS 1 +#endif + +#ifndef __TBB_STRICT_CONSTRAINTS +#define __TBB_STRICT_CONSTRAINTS 1 +#endif + +#if __TBB_CPP20_CONCEPTS_PRESENT && __TBB_USE_CONSTRAINTS + #define __TBB_requires(...) requires __VA_ARGS__ +#else // __TBB_CPP20_CONCEPTS_PRESENT + #define __TBB_requires(...) +#endif // __TBB_CPP20_CONCEPTS_PRESENT + +/** Macros of the form __TBB_XXX_BROKEN denote known issues that are caused by + the bugs in compilers, standard or OS specific libraries. They should be + removed as soon as the corresponding bugs are fixed or the buggy OS/compiler + versions go out of the support list. +**/ + +// Some STL containers not support allocator traits in old GCC versions +#if __GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION <= 50301 + #define TBB_ALLOCATOR_TRAITS_BROKEN 1 +#endif + +// GCC 4.8 C++ standard library implements std::this_thread::yield as no-op. +#if __TBB_GLIBCXX_VERSION >= 40800 && __TBB_GLIBCXX_VERSION < 40900 + #define __TBB_GLIBCXX_THIS_THREAD_YIELD_BROKEN 1 +#endif + +/** End of __TBB_XXX_BROKEN macro section **/ + +#if defined(_MSC_VER) && _MSC_VER>=1500 && !defined(__INTEL_COMPILER) + // A macro to suppress erroneous or benign "unreachable code" MSVC warning (4702) + #define __TBB_MSVC_UNREACHABLE_CODE_IGNORED 1 +#endif + +// Many OS versions (Android 4.0.[0-3] for example) need workaround for dlopen to avoid non-recursive loader lock hang +// Setting the workaround for all compile targets ($APP_PLATFORM) below Android 4.4 (android-19) +#if __ANDROID__ + #include +#endif + +#ifndef __TBB_CRITICAL_TASKS +#define __TBB_CRITICAL_TASKS 1 +#endif + +#define __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING (TBB_PREVIEW_FLOW_GRAPH_FEATURES) + + +#ifndef __TBB_PREVIEW_FLOW_GRAPH_NODE_SET +#define __TBB_PREVIEW_FLOW_GRAPH_NODE_SET (TBB_PREVIEW_FLOW_GRAPH_FEATURES) +#endif + +#ifndef __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +#define __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT (TBB_PREVIEW_FLOW_GRAPH_FEATURES \ + || TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT) +#endif + +#ifndef __TBB_PREVIEW_FLOW_GRAPH_RESOURCE_LIMITING +#define __TBB_PREVIEW_FLOW_GRAPH_RESOURCE_LIMITING (TBB_PREVIEW_FLOW_GRAPH_FEATURES \ + || TBB_PREVIEW_FLOW_GRAPH_RESOURCE_LIMITING) +#endif + +#if TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS +#define __TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS 1 +#endif + +#if TBB_PREVIEW_TASK_GROUP_EXTENSIONS || __TBB_BUILD +#define __TBB_PREVIEW_TASK_GROUP_EXTENSIONS 1 +#endif + +#if TBB_PREVIEW_PARALLEL_PHASE || __TBB_BUILD +#define __TBB_PREVIEW_PARALLEL_PHASE 1 +#endif + +#if TBB_PREVIEW_TASK_ARENA_CORE_TYPE_SELECTOR || __TBB_BUILD +#define __TBB_PREVIEW_TASK_ARENA_CORE_TYPE_SELECTOR 1 +#endif + +#if !__TBB_DISABLE_SPEC_EXTENSIONS +#define TBB_EXT_CUSTOM_ASSERTION_HANDLER 202510 +#endif + +// Feature-test macros +#if __TBB_PREVIEW_FLOW_GRAPH_RESOURCE_LIMITING +#define TBB_HAS_FLOW_GRAPH_RESOURCE_LIMITING 202603 +#endif + +#if __TBB_PREVIEW_PARALLEL_PHASE +#define TBB_HAS_PARALLEL_PHASE 202603 +#endif + +#if __TBB_PREVIEW_TASK_ARENA_CORE_TYPE_SELECTOR +#define TBB_HAS_TASK_ARENA_CORE_TYPE_SELECTOR 202603 +#endif + +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS +#define TBB_HAS_TASK_GROUP_DEPENDENCIES 202603 +#endif + +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS +#define TBB_HAS_TASK_GROUP_WAIT_FOR_SINGLE_TASK 202603 +#endif + +#endif // __TBB_detail__config_H diff --git a/src/3rd party/oneapi/tbb/detail/_containers_helpers.h b/src/3rd party/oneapi/tbb/detail/_containers_helpers.h new file mode 100644 index 0000000000..4dca07fa10 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_containers_helpers.h @@ -0,0 +1,67 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__containers_helpers_H +#define __TBB_detail__containers_helpers_H + +#include "_template_helpers.h" +#include "_allocator_traits.h" +#include +#include +#include + +namespace tbb { +namespace detail { +inline namespace d0 { + +template +struct comp_is_transparent : std::false_type {}; + +template +struct comp_is_transparent> : std::true_type {}; + +template +struct has_transparent_key_equal : std::false_type { using type = KeyEqual; }; + +template +struct has_transparent_key_equal> : std::true_type { + using type = typename Hasher::transparent_key_equal; + static_assert(comp_is_transparent::value, "Hash::transparent_key_equal::is_transparent is not valid or does not denote a type."); + static_assert((std::is_same>::value || + std::is_same::value), "KeyEqual is a different type than equal_to or Hash::transparent_key_equal."); + }; + +struct is_iterator_impl { +template +using iter_traits_category = typename std::iterator_traits::iterator_category; + +template +using input_iter_category = typename std::enable_if>::value>::type; +}; // struct is_iterator_impl + +template +using is_input_iterator = supports; + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +template +inline constexpr bool is_input_iterator_v = is_input_iterator::value; +#endif + +} // inline namespace d0 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__containers_helpers_H diff --git a/src/3rd party/oneapi/tbb/detail/_exception.h b/src/3rd party/oneapi/tbb/detail/_exception.h new file mode 100644 index 0000000000..2cf93e2824 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_exception.h @@ -0,0 +1,95 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__exception_H +#define __TBB__exception_H + +#include "_config.h" + +#include // std::bad_alloc +#include // std::exception +#include // std::runtime_error + +namespace tbb { +namespace detail { +inline namespace d0 { +enum class exception_id { + bad_alloc = 1, + bad_last_alloc, + user_abort, + nonpositive_step, + out_of_range, + reservation_length_error, + missing_wait, + invalid_load_factor, + invalid_key, + bad_tagged_msg_cast, + unsafe_wait, + last_entry +}; +} // namespace d0 + +#if _MSC_VER + #pragma warning(disable: 4275) +#endif + +namespace r1 { +//! Exception for concurrent containers +class TBB_EXPORT bad_last_alloc : public std::bad_alloc { +public: + const char* __TBB_EXPORTED_METHOD what() const noexcept(true) override; +}; + +//! Exception for user-initiated abort +class TBB_EXPORT user_abort : public std::exception { +public: + const char* __TBB_EXPORTED_METHOD what() const noexcept(true) override; +}; + +//! Exception for missing wait on structured_task_group +class TBB_EXPORT missing_wait : public std::exception { +public: + const char* __TBB_EXPORTED_METHOD what() const noexcept(true) override; +}; + +//! Exception for impossible finalization of task_sheduler_handle +#if __APPLE__ || __FreeBSD__ + #pragma GCC visibility push(default) +#endif +class TBB_EXPORT unsafe_wait : public std::runtime_error { +public: + unsafe_wait(const char* msg) : std::runtime_error(msg) {} +}; +#if __APPLE__ || __FreeBSD__ + #pragma GCC visibility pop +#endif + +//! Gathers all throw operators in one place. +/** Its purpose is to minimize code bloat that can be caused by throw operators + scattered in multiple places, especially in templates. **/ +TBB_EXPORT void __TBB_EXPORTED_FUNC throw_exception ( exception_id ); +} // namespace r1 + +inline namespace d0 { +using r1::throw_exception; +} // namespace d0 + +} // namespace detail +} // namespace tbb + +#endif // __TBB__exception_H + diff --git a/src/3rd party/oneapi/tbb/detail/_export.h b/src/3rd party/oneapi/tbb/detail/_export.h new file mode 100644 index 0000000000..24b6c08efb --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_export.h @@ -0,0 +1,54 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__export_H +#define __TBB_detail__export_H + +#if defined(__MINGW32__) + #define __TBB_EXPORT __declspec(dllexport) +#elif defined(_WIN32) // Use .def files for these + #define __TBB_EXPORT +#elif defined(__unix__) || defined(__APPLE__) // Use .def files for these + #define __TBB_EXPORT __attribute__ ((visibility ("default"))) +#else + #error "Unknown platform/compiler" +#endif + +#if __TBB_BUILD + #define TBB_EXPORT __TBB_EXPORT +#else + #define TBB_EXPORT +#endif + +#if __TBBMALLOC_BUILD + #define TBBMALLOC_EXPORT __TBB_EXPORT +#else + #define TBBMALLOC_EXPORT +#endif + +#if __TBBMALLOCPROXY_BUILD + #define TBBMALLOCPROXY_EXPORT __TBB_EXPORT +#else + #define TBBMALLOCPROXY_EXPORT +#endif + +#if __TBBBIND_BUILD + #define TBBBIND_EXPORT __TBB_EXPORT +#else + #define TBBBIND_EXPORT +#endif + +#endif diff --git a/src/3rd party/tbb/internal/_flow_graph_body_impl.h b/src/3rd party/oneapi/tbb/detail/_flow_graph_body_impl.h similarity index 51% rename from src/3rd party/tbb/internal/_flow_graph_body_impl.h rename to src/3rd party/oneapi/tbb/detail/_flow_graph_body_impl.h index 15c3b92146..21da06ce03 100644 --- a/src/3rd party/tbb/internal/_flow_graph_body_impl.h +++ b/src/3rd party/oneapi/tbb/detail/_flow_graph_body_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,48 +21,28 @@ #error Do not #include this internal file directly; use public TBB headers instead. #endif -// included in namespace tbb::flow::interfaceX (in flow_graph.h) +// included in namespace tbb::detail::d2 (in flow_graph.h) -namespace internal { +typedef std::uint64_t tag_value; -typedef tbb::internal::uint64_t tag_value; - -using tbb::internal::strip; - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT +// TODO revamp: find out if there is already helper for has_policy. template struct Policy {}; template struct has_policy; template struct has_policy : - tbb::internal::bool_constant::value || + std::integral_constant::value || has_policy::value> {}; template struct has_policy : - tbb::internal::bool_constant::value> {}; + std::integral_constant::value> {}; template struct has_policy > : has_policy {}; -#else - -template struct Policy {}; - -template -struct has_policy : tbb::internal::bool_constant::value> {}; - -template -struct has_policy > : has_policy {}; - -template -struct has_policy > : - tbb::internal::bool_constant::value || has_policy::value> {}; - -#endif - namespace graph_policy_namespace { struct rejecting { }; @@ -73,10 +53,11 @@ namespace graph_policy_namespace { // K == type of field used for key-matching. Each tag-matching port will be provided // functor that, given an object accepted by the port, will return the /// field of type K being used for matching. - template::type > > + template::type > > + __TBB_requires(tbb::detail::hash_compare) struct key_matching { typedef K key_type; - typedef typename strip::type base_key_type; + typedef typename std::decay::type base_key_type; typedef KHash hash_compare_type; }; @@ -84,8 +65,8 @@ namespace graph_policy_namespace { typedef key_matching tag_matching; // Aliases for Policy combinations - typedef interface11::internal::Policy queueing_lightweight; - typedef interface11::internal::Policy rejecting_lightweight; + typedef Policy queueing_lightweight; + typedef Policy rejecting_lightweight; } // namespace graph_policy_namespace @@ -93,21 +74,21 @@ namespace graph_policy_namespace { //! A functor that takes no input and generates a value of type Output template< typename Output > -class source_body : tbb::internal::no_assign { +class input_body : no_assign { public: - virtual ~source_body() {} - virtual bool operator()(Output &output) = 0; - virtual source_body* clone() = 0; + virtual ~input_body() {} + virtual Output operator()(d1::flow_control& fc) = 0; + virtual input_body* clone() = 0; }; -//! The leaf for source_body +//! The leaf for input_body template< typename Output, typename Body> -class source_body_leaf : public source_body { +class input_body_leaf : public input_body { public: - source_body_leaf( const Body &_body ) : body(_body) { } - bool operator()(Output &output) __TBB_override { return body( output ); } - source_body_leaf* clone() __TBB_override { - return new source_body_leaf< Output, Body >(body); + input_body_leaf( const Body &_body ) : body(_body) { } + Output operator()(d1::flow_control& fc) override { return body(fc); } + input_body_leaf* clone() override { + return new input_body_leaf< Output, Body >(body); } Body get_body() { return body; } private: @@ -116,7 +97,7 @@ class source_body_leaf : public source_body { //! A functor that takes an Input and generates an Output template< typename Input, typename Output > -class function_body : tbb::internal::no_assign { +class function_body : no_assign { public: virtual ~function_body() {} virtual Output operator()(const Input &input) = 0; @@ -128,9 +109,9 @@ template class function_body_leaf : public function_body< Input, Output > { public: function_body_leaf( const B &_body ) : body(_body) { } - Output operator()(const Input &i) __TBB_override { return body(i); } + Output operator()(const Input &i) override { return tbb::detail::invoke(body,i); } B get_body() { return body; } - function_body_leaf* clone() __TBB_override { + function_body_leaf* clone() override { return new function_body_leaf< Input, Output, B >(body); } private: @@ -142,12 +123,12 @@ template class function_body_leaf< continue_msg, continue_msg, B> : public function_body< continue_msg, continue_msg > { public: function_body_leaf( const B &_body ) : body(_body) { } - continue_msg operator()( const continue_msg &i ) __TBB_override { + continue_msg operator()( const continue_msg &i ) override { body(i); return i; } B get_body() { return body; } - function_body_leaf* clone() __TBB_override { + function_body_leaf* clone() override { return new function_body_leaf< continue_msg, continue_msg, B >(body); } private: @@ -159,12 +140,12 @@ template class function_body_leaf< Input, continue_msg, B> : public function_body< Input, continue_msg > { public: function_body_leaf( const B &_body ) : body(_body) { } - continue_msg operator()(const Input &i) __TBB_override { + continue_msg operator()(const Input &i) override { body(i); return continue_msg(); } B get_body() { return body; } - function_body_leaf* clone() __TBB_override { + function_body_leaf* clone() override { return new function_body_leaf< Input, continue_msg, B >(body); } private: @@ -176,11 +157,11 @@ template class function_body_leaf< continue_msg, Output, B > : public function_body< continue_msg, Output > { public: function_body_leaf( const B &_body ) : body(_body) { } - Output operator()(const continue_msg &i) __TBB_override { + Output operator()(const continue_msg &i) override { return body(i); } B get_body() { return body; } - function_body_leaf* clone() __TBB_override { + function_body_leaf* clone() override { return new function_body_leaf< continue_msg, Output, B >(body); } private: @@ -189,7 +170,7 @@ class function_body_leaf< continue_msg, Output, B > : public function_body< cont //! function_body that takes an Input and a set of output ports template -class multifunction_body : tbb::internal::no_assign { +class multifunction_body : no_assign { public: virtual ~multifunction_body () {} virtual void operator()(const Input &/* input*/, OutputSet &/*oset*/) = 0; @@ -202,11 +183,11 @@ template class multifunction_body_leaf : public multifunction_body { public: multifunction_body_leaf(const B &_body) : body(_body) { } - void operator()(const Input &input, OutputSet &oset) __TBB_override { - body(input, oset); // body may explicitly put() to one or more of oset. + void operator()(const Input &input, OutputSet &oset) override { + tbb::detail::invoke(body, input, oset); // body may explicitly put() to one or more of oset. } - void* get_body_ptr() __TBB_override { return &body; } - multifunction_body_leaf* clone() __TBB_override { + void* get_body_ptr() override { return &body; } + multifunction_body_leaf* clone() override { return new multifunction_body_leaf(body); } @@ -217,7 +198,7 @@ class multifunction_body_leaf : public multifunction_body { // ------ function bodies for hash_buffers and key-matching joins. template -class type_to_key_function_body : tbb::internal::no_assign { +class type_to_key_function_body : no_assign { public: virtual ~type_to_key_function_body() {} virtual Output operator()(const Input &input) = 0; // returns an Output @@ -226,7 +207,7 @@ class type_to_key_function_body : tbb::internal::no_assign { // specialization for ref output template -class type_to_key_function_body : tbb::internal::no_assign { +class type_to_key_function_body : no_assign { public: virtual ~type_to_key_function_body() {} virtual const Output & operator()(const Input &input) = 0; // returns a const Output& @@ -237,9 +218,8 @@ template class type_to_key_function_body_leaf : public type_to_key_function_body { public: type_to_key_function_body_leaf( const B &_body ) : body(_body) { } - Output operator()(const Input &i) __TBB_override { return body(i); } - B get_body() { return body; } - type_to_key_function_body_leaf* clone() __TBB_override { + Output operator()(const Input &i) override { return tbb::detail::invoke(body, i); } + type_to_key_function_body_leaf* clone() override { return new type_to_key_function_body_leaf< Input, Output, B>(body); } private: @@ -250,11 +230,10 @@ template class type_to_key_function_body_leaf : public type_to_key_function_body< Input, Output&> { public: type_to_key_function_body_leaf( const B &_body ) : body(_body) { } - const Output& operator()(const Input &i) __TBB_override { - return body(i); + const Output& operator()(const Input &i) override { + return tbb::detail::invoke(body, i); } - B get_body() { return body; } - type_to_key_function_body_leaf* clone() __TBB_override { + type_to_key_function_body_leaf* clone() override { return new type_to_key_function_body_leaf< Input, Output&, B>(body); } private: @@ -268,162 +247,171 @@ class type_to_key_function_body_leaf : public type_to_key_funct //! A task that calls a node's forward_task function template< typename NodeType > class forward_task_bypass : public graph_task { - NodeType &my_node; - public: - - forward_task_bypass( NodeType &n -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES + forward_task_bypass( graph& g, d1::small_object_allocator& allocator, NodeType &n , node_priority_t node_priority = no_priority - ) : graph_task(node_priority), -#else - ) : -#endif + ) : graph_task(g, allocator, node_priority), my_node(n) {} - task *execute() __TBB_override { - task * new_task = my_node.forward_task(); - if (new_task == SUCCESSFULLY_ENQUEUED) new_task = NULL; - return new_task; + d1::task* execute(d1::execution_data& ed) override { + graph_task* next_task = my_node.forward_task(); + if (SUCCESSFULLY_ENQUEUED == next_task) + next_task = nullptr; + else if (next_task) + next_task = prioritize_task(my_node.graph_reference(), *next_task); + finalize(ed); + return next_task; + } + + d1::task* cancel(d1::execution_data& ed) override { + finalize(ed); + return nullptr; } }; //! A task that calls a node's apply_body_bypass function, passing in an input of type Input -// return the task* unless it is SUCCESSFULLY_ENQUEUED, in which case return NULL -template< typename NodeType, typename Input > -class apply_body_task_bypass : public graph_task { - +// return the task* unless it is SUCCESSFULLY_ENQUEUED, in which case return nullptr +template< typename NodeType, typename Input, typename BaseTaskType = graph_task> +class apply_body_task_bypass + : public BaseTaskType +{ NodeType &my_node; Input my_input; -public: + using check_metainfo = std::is_same; + using without_metainfo = std::true_type; + using with_metainfo = std::false_type; - apply_body_task_bypass( NodeType &n, const Input &i -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES - , node_priority_t node_priority = no_priority - ) : graph_task(node_priority), -#else - ) : + graph_task* call_apply_body_bypass_impl(without_metainfo) { + return my_node.apply_body_bypass(my_input + __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* call_apply_body_bypass_impl(with_metainfo) { + return my_node.apply_body_bypass(my_input, message_metainfo{this->get_msg_wait_context_vertices()}); + } #endif - my_node(n), my_input(i) {} - task *execute() __TBB_override { - task * next_task = my_node.apply_body_bypass( my_input ); - if(next_task == SUCCESSFULLY_ENQUEUED) next_task = NULL; + graph_task* call_apply_body_bypass() { + return call_apply_body_bypass_impl(check_metainfo{}); + } + +public: +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + apply_body_task_bypass( graph& g, d1::small_object_allocator& allocator, NodeType &n, const Input &i, + node_priority_t node_priority, Metainfo&& metainfo ) + : BaseTaskType(g, allocator, node_priority, std::forward(metainfo).waiters()) + , my_node(n), my_input(i) {} +#endif + + apply_body_task_bypass( graph& g, d1::small_object_allocator& allocator, NodeType& n, const Input& i, + node_priority_t node_priority = no_priority ) + : BaseTaskType(g, allocator, node_priority), my_node(n), my_input(i) {} + + d1::task* execute(d1::execution_data& ed) override { + graph_task* next_task = call_apply_body_bypass(); + if (SUCCESSFULLY_ENQUEUED == next_task) + next_task = nullptr; + else if (next_task) + next_task = prioritize_task(my_node.graph_reference(), *next_task); + BaseTaskType::template finalize(ed); return next_task; } + + d1::task* cancel(d1::execution_data& ed) override { + BaseTaskType::template finalize(ed); + return nullptr; + } }; //! A task that calls a node's apply_body_bypass function with no input template< typename NodeType > -class source_task_bypass : public graph_task { - +class input_node_task_bypass : public graph_task { NodeType &my_node; - public: + input_node_task_bypass( graph& g, d1::small_object_allocator& allocator, NodeType &n ) + : graph_task(g, allocator), my_node(n) {} + + d1::task* execute(d1::execution_data& ed) override { + graph_task* next_task = my_node.apply_body_bypass( ); + if (SUCCESSFULLY_ENQUEUED == next_task) + next_task = nullptr; + else if (next_task) + next_task = prioritize_task(my_node.graph_reference(), *next_task); + finalize(ed); + return next_task; + } - source_task_bypass( NodeType &n ) : my_node(n) {} - - task *execute() __TBB_override { - task *new_task = my_node.apply_body_bypass( ); - if(new_task == SUCCESSFULLY_ENQUEUED) return NULL; - return new_task; + d1::task* cancel(d1::execution_data& ed) override { + finalize(ed); + return nullptr; } }; // ------------------------ end of node task bodies ----------------------------------- -//! An empty functor that takes an Input and returns a default constructed Output -template< typename Input, typename Output > -struct empty_body { - Output operator()( const Input & ) const { return Output(); } -}; - template -class decrementer; +class threshold_regulator; template -class decrementer::value, void>::type - > : public receiver, tbb::internal::no_copy { +class threshold_regulator::value>::type> + : public receiver, no_copy +{ T* my_node; protected: - task* try_put_task( const DecrementType& value ) __TBB_override { - task* result = my_node->decrement_counter( value ); + graph_task* try_put_task( const DecrementType& value ) override { + graph_task* result = my_node->decrement_counter( value ); if( !result ) result = SUCCESSFULLY_ENQUEUED; return result; } - graph& graph_reference() const __TBB_override { - return my_node->my_graph; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + // Intentionally ignore the metainformation + // If there are more items associated with passed metainfo to be processed + // They should be stored in the buffer before the limiter_node + graph_task* try_put_task(const DecrementType& value, const message_metainfo&) override { + return try_put_task(value); } - - template friend class tbb::flow::interface11::limiter_node; - void reset_receiver( reset_flags f ) __TBB_override { -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - if (f & rf_clear_edges) - my_built_predecessors.clear(); -#else - tbb::internal::suppress_unused_warning( f ); #endif - } -public: - // Since decrementer does not make use of possibly unconstructed owner inside its - // constructor, my_node can be directly initialized with 'this' pointer passed from the - // owner, hence making method 'set_owner' needless. - decrementer() : my_node(NULL) {} - void set_owner( T *node ) { my_node = node; } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - spin_mutex my_mutex; - //! The predecessor type for this node - typedef typename receiver::predecessor_type predecessor_type; - - typedef internal::edge_container built_predecessors_type; - typedef typename built_predecessors_type::edge_list_type predecessor_list_type; - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - void internal_add_built_predecessor( predecessor_type &s) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - my_built_predecessors.add_edge( s ); - } - - void internal_delete_built_predecessor( predecessor_type &s) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - my_built_predecessors.delete_edge(s); + graph& graph_reference() const override { + return my_node->my_graph; } - void copy_predecessors( predecessor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - my_built_predecessors.copy_edges(v); - } + template friend class limiter_node; + void reset_receiver( reset_flags ) {} - size_t predecessor_count() __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - return my_built_predecessors.edge_count(); +public: + threshold_regulator(T* owner) : my_node(owner) { + // Do not work with the passed pointer here as it may not be fully initialized yet } -protected: - built_predecessors_type my_built_predecessors; -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ }; template -class decrementer : public continue_receiver, tbb::internal::no_copy { +class threshold_regulator : public continue_receiver, no_copy { T *my_node; - task *execute() __TBB_override { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + // Intentionally ignore the metainformation + // If there are more items associated with passed metainfo to be processed + // They should be stored in the buffer before the limiter_node + graph_task* execute(const message_metainfo&) override { +#else + graph_task* execute() override { +#endif return my_node->decrement_counter( 1 ); } protected: - graph& graph_reference() const __TBB_override { + graph& graph_reference() const override { return my_node->my_graph; } @@ -431,19 +419,11 @@ class decrementer : public continue_receiver, tbb::intern typedef continue_msg input_type; typedef continue_msg output_type; - decrementer( int number_of_predecessors = 0 ) - : continue_receiver( - __TBB_FLOW_GRAPH_PRIORITY_ARG1(number_of_predecessors, tbb::flow::internal::no_priority) - ) - // Since decrementer does not make use of possibly unconstructed owner inside its - // constructor, my_node can be directly initialized with 'this' pointer passed from the - // owner, hence making method 'set_owner' needless. - , my_node(NULL) - {} - void set_owner( T *node ) { my_node = node; } + threshold_regulator(T* owner) + : continue_receiver( /*number_of_predecessors=*/0, no_priority ), my_node(owner) + { + // Do not work with the passed pointer here as it may not be fully initialized yet + } }; -} // namespace internal - #endif // __TBB__flow_graph_body_impl_H - diff --git a/src/3rd party/oneapi/tbb/detail/_flow_graph_cache_impl.h b/src/3rd party/oneapi/tbb/detail/_flow_graph_cache_impl.h new file mode 100644 index 0000000000..647f3dc1b6 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_flow_graph_cache_impl.h @@ -0,0 +1,501 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__flow_graph_cache_impl_H +#define __TBB__flow_graph_cache_impl_H + +#ifndef __TBB_flow_graph_H +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +// included in namespace tbb::detail::d2 (in flow_graph.h) + +//! A node_cache maintains a std::queue of elements of type T. Each operation is protected by a lock. +template< typename T, typename M=spin_mutex > +class node_cache { + public: + + typedef size_t size_type; + + bool empty() { + typename mutex_type::scoped_lock lock( my_mutex ); + return internal_empty(); + } + + void add( T &n ) { + typename mutex_type::scoped_lock lock( my_mutex ); + internal_push(n); + } + + void remove( T &n ) { + typename mutex_type::scoped_lock lock( my_mutex ); + for ( size_t i = internal_size(); i != 0; --i ) { + T &s = internal_pop(); + if ( &s == &n ) + break; // only remove one predecessor per request + internal_push(s); + } + } + + void clear() { + while( !my_q.empty()) (void)my_q.pop(); + } + +protected: + + typedef M mutex_type; + mutex_type my_mutex; + std::queue< T * > my_q; + + // Assumes lock is held + inline bool internal_empty( ) { + return my_q.empty(); + } + + // Assumes lock is held + inline size_type internal_size( ) { + return my_q.size(); + } + + // Assumes lock is held + inline void internal_push( T &n ) { + my_q.push(&n); + } + + // Assumes lock is held + inline T &internal_pop() { + T *v = my_q.front(); + my_q.pop(); + return *v; + } + +}; + +//! A cache of predecessors that only supports try_get +template< typename T, typename M=spin_mutex > +class predecessor_cache : public node_cache< sender, M > { +public: + typedef M mutex_type; + typedef T output_type; + typedef sender predecessor_type; + typedef receiver successor_type; + + predecessor_cache( successor_type* owner ) : my_owner( owner ) { + __TBB_ASSERT( my_owner, "predecessor_cache should have an owner." ); + // Do not work with the passed pointer here as it may not be fully initialized yet + } + +private: + bool get_item_impl( output_type& v + __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo* metainfo_ptr = nullptr) ) + { + + bool successful_get = false; + + do { + predecessor_type *src; + { + typename mutex_type::scoped_lock lock(this->my_mutex); + if ( this->internal_empty() ) { + break; + } + src = &this->internal_pop(); + } + + // Try to get from this sender +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (metainfo_ptr) { + successful_get = src->try_get( v, *metainfo_ptr ); + } else +#endif + { + successful_get = src->try_get( v ); + } + + if (successful_get == false) { + // Relinquish ownership of the edge + register_successor(*src, *my_owner); + } else { + // Retain ownership of the edge + this->add(*src); + } + } while ( successful_get == false ); + return successful_get; + } +public: + bool get_item( output_type& v ) { + return get_item_impl(v); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool get_item( output_type& v, message_metainfo& metainfo ) { + return get_item_impl(v, &metainfo); + } +#endif + + // If we are removing arcs (rf_clear_edges), call clear() rather than reset(). + void reset() { + for(;;) { + predecessor_type *src; + { + if (this->internal_empty()) break; + src = &this->internal_pop(); + } + register_successor(*src, *my_owner); + } + } + +protected: + successor_type* my_owner; +}; + +//! An cache of predecessors that supports requests and reservations +template< typename T, typename M=spin_mutex > +class reservable_predecessor_cache : public predecessor_cache< T, M > { +public: + typedef M mutex_type; + typedef T output_type; + typedef sender predecessor_type; + typedef receiver successor_type; + + reservable_predecessor_cache( successor_type* owner ) + : predecessor_cache(owner), reserved_src(nullptr) + { + // Do not work with the passed pointer here as it may not be fully initialized yet + } + +private: + bool try_reserve_impl( output_type &v __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo* metainfo) ) { + bool successful_reserve = false; + + do { + predecessor_type* pred = nullptr; + { + typename mutex_type::scoped_lock lock(this->my_mutex); + if ( reserved_src.load(std::memory_order_relaxed) || this->internal_empty() ) + return false; + + pred = &this->internal_pop(); + reserved_src.store(pred, std::memory_order_relaxed); + } + + // Try to get from this sender +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (metainfo) { + successful_reserve = pred->try_reserve( v, *metainfo ); + } else +#endif + { + successful_reserve = pred->try_reserve( v ); + } + + if (successful_reserve == false) { + typename mutex_type::scoped_lock lock(this->my_mutex); + // Relinquish ownership of the edge + register_successor( *pred, *this->my_owner ); + reserved_src.store(nullptr, std::memory_order_relaxed); + } else { + // Retain ownership of the edge + this->add( *pred); + } + } while ( successful_reserve == false ); + + return successful_reserve; + } +public: + bool try_reserve( output_type& v ) { + return try_reserve_impl(v __TBB_FLOW_GRAPH_METAINFO_ARG(nullptr)); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool try_reserve( output_type& v, message_metainfo& metainfo ) { + return try_reserve_impl(v, &metainfo); + } +#endif + + bool try_release() { + reserved_src.load(std::memory_order_relaxed)->try_release(); + reserved_src.store(nullptr, std::memory_order_relaxed); + return true; + } + + bool try_consume() { + reserved_src.load(std::memory_order_relaxed)->try_consume(); + reserved_src.store(nullptr, std::memory_order_relaxed); + return true; + } + + void reset() { + reserved_src.store(nullptr, std::memory_order_relaxed); + predecessor_cache::reset(); + } + + void clear() { + reserved_src.store(nullptr, std::memory_order_relaxed); + predecessor_cache::clear(); + } + +private: + std::atomic reserved_src; +}; + + +//! An abstract cache of successors +template +class successor_cache : no_copy { +protected: + + typedef M mutex_type; + mutex_type my_mutex; + + typedef receiver successor_type; + typedef receiver* pointer_type; + typedef sender owner_type; + // TODO revamp: introduce heapified collection of successors for strict priorities + typedef std::list< pointer_type > successors_type; + successors_type my_successors; + + owner_type* my_owner; + +public: + successor_cache( owner_type* owner ) : my_owner(owner) { + // Do not work with the passed pointer here as it may not be fully initialized yet + } + + virtual ~successor_cache() {} + + void register_successor( successor_type& r ) { + typename mutex_type::scoped_lock l(my_mutex, true); + if( r.priority() != no_priority ) + my_successors.push_front( &r ); + else + my_successors.push_back( &r ); + } + + void remove_successor( successor_type& r ) { + typename mutex_type::scoped_lock l(my_mutex, true); + for ( typename successors_type::iterator i = my_successors.begin(); + i != my_successors.end(); ++i ) { + if ( *i == & r ) { + my_successors.erase(i); + break; + } + } + } + + bool empty() { + typename mutex_type::scoped_lock l(my_mutex, false); + return my_successors.empty(); + } + + void clear() { + my_successors.clear(); + } + + virtual graph_task* try_put_task( const T& t ) = 0; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + virtual graph_task* try_put_task( const T& t, const message_metainfo& metainfo ) = 0; +#endif +}; // successor_cache + +//! An abstract cache of successors, specialized to continue_msg +template +class successor_cache< continue_msg, M > : no_copy { +protected: + + typedef M mutex_type; + mutex_type my_mutex; + + typedef receiver successor_type; + typedef receiver* pointer_type; + typedef sender owner_type; + typedef std::list< pointer_type > successors_type; + successors_type my_successors; + owner_type* my_owner; + +public: + successor_cache( sender* owner ) : my_owner(owner) { + // Do not work with the passed pointer here as it may not be fully initialized yet + } + + virtual ~successor_cache() {} + + void register_successor( successor_type& r ) { + typename mutex_type::scoped_lock l(my_mutex, true); + if( r.priority() != no_priority ) + my_successors.push_front( &r ); + else + my_successors.push_back( &r ); + __TBB_ASSERT( my_owner, "Cache of successors must have an owner." ); + if ( r.is_continue_receiver() ) { + r.register_predecessor( *my_owner ); + } + } + + void remove_successor( successor_type& r ) { + typename mutex_type::scoped_lock l(my_mutex, true); + for ( successors_type::iterator i = my_successors.begin(); i != my_successors.end(); ++i ) { + if ( *i == &r ) { + __TBB_ASSERT(my_owner, "Cache of successors must have an owner."); + // TODO: check if we need to test for continue_receiver before removing from r. + r.remove_predecessor( *my_owner ); + my_successors.erase(i); + break; + } + } + } + + bool empty() { + typename mutex_type::scoped_lock l(my_mutex, false); + return my_successors.empty(); + } + + void clear() { + my_successors.clear(); + } + + virtual graph_task* try_put_task( const continue_msg& t ) = 0; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + virtual graph_task* try_put_task( const continue_msg& t, const message_metainfo& metainfo ) = 0; +#endif +}; // successor_cache< continue_msg > + +//! A cache of successors that are broadcast to +template +class broadcast_cache : public successor_cache { + typedef successor_cache base_type; + typedef M mutex_type; + typedef typename successor_cache::successors_type successors_type; + + graph_task* try_put_task_impl( const T& t __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo) ) { + graph_task * last_task = nullptr; + typename mutex_type::scoped_lock l(this->my_mutex, /*write=*/true); + typename successors_type::iterator i = this->my_successors.begin(); + while ( i != this->my_successors.end() ) { + graph_task *new_task = (*i)->try_put_task(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + // workaround for icc bug + graph& graph_ref = (*i)->graph_reference(); + last_task = combine_tasks(graph_ref, last_task, new_task); // enqueue if necessary + if(new_task) { + ++i; + } + else { // failed + if ( (*i)->register_predecessor(*this->my_owner) ) { + i = this->my_successors.erase(i); + } else { + ++i; + } + } + } + return last_task; + } +public: + + broadcast_cache( typename base_type::owner_type* owner ): base_type(owner) { + // Do not work with the passed pointer here as it may not be fully initialized yet + } + + graph_task* try_put_task( const T &t ) override { + return try_put_task_impl(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task( const T &t, const message_metainfo& metainfo ) override { + return try_put_task_impl(t, metainfo); + } +#endif + + // call try_put_task and return list of received tasks + bool gather_successful_try_puts( const T &t, graph_task_list& tasks ) { + bool is_at_least_one_put_successful = false; + typename mutex_type::scoped_lock l(this->my_mutex, /*write=*/true); + typename successors_type::iterator i = this->my_successors.begin(); + while ( i != this->my_successors.end() ) { + graph_task * new_task = (*i)->try_put_task(t); + if(new_task) { + ++i; + if(new_task != SUCCESSFULLY_ENQUEUED) { + tasks.push_back(*new_task); + } + is_at_least_one_put_successful = true; + } + else { // failed + if ( (*i)->register_predecessor(*this->my_owner) ) { + i = this->my_successors.erase(i); + } else { + ++i; + } + } + } + return is_at_least_one_put_successful; + } +}; + +//! A cache of successors that are put in a round-robin fashion +template +class round_robin_cache : public successor_cache { + typedef successor_cache base_type; + typedef size_t size_type; + typedef M mutex_type; + typedef typename successor_cache::successors_type successors_type; + +public: + + round_robin_cache( typename base_type::owner_type* owner ): base_type(owner) { + // Do not work with the passed pointer here as it may not be fully initialized yet + } + + size_type size() { + typename mutex_type::scoped_lock l(this->my_mutex, false); + return this->my_successors.size(); + } + +private: + + graph_task* try_put_task_impl( const T &t + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo) ) + { + typename mutex_type::scoped_lock l(this->my_mutex, /*write=*/true); + typename successors_type::iterator i = this->my_successors.begin(); + while ( i != this->my_successors.end() ) { + graph_task* new_task = (*i)->try_put_task(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + if ( new_task ) { + return new_task; + } else { + if ( (*i)->register_predecessor(*this->my_owner) ) { + i = this->my_successors.erase(i); + } + else { + ++i; + } + } + } + return nullptr; + } + +public: + graph_task* try_put_task(const T& t) override { + return try_put_task_impl(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task( const T& t, const message_metainfo& metainfo ) override { + return try_put_task_impl(t, metainfo); + } +#endif +}; + +#endif // __TBB__flow_graph_cache_impl_H diff --git a/src/3rd party/oneapi/tbb/detail/_flow_graph_impl.h b/src/3rd party/oneapi/tbb/detail/_flow_graph_impl.h new file mode 100644 index 0000000000..86f3d5888d --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_flow_graph_impl.h @@ -0,0 +1,556 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + Copyright (c) 2025 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_flow_graph_impl_H +#define __TBB_flow_graph_impl_H + +#include "_config.h" +#include "_task.h" +#include "../task_group.h" +#include "../task_arena.h" +#include "../flow_graph_abstractions.h" + +#include "../concurrent_priority_queue.h" + +#include + +namespace tbb { +namespace detail { + +namespace d2 { + +class graph_task; +__TBB_GLOBAL_VAR graph_task* const SUCCESSFULLY_ENQUEUED = (graph_task*)-1; +typedef unsigned int node_priority_t; +__TBB_GLOBAL_VAR constexpr node_priority_t no_priority = node_priority_t(0); + +class graph; +class graph_node; + +template +class graph_iterator { + friend class graph; + friend class graph_node; +public: + typedef size_t size_type; + typedef GraphNodeType value_type; + typedef GraphNodeType* pointer; + typedef GraphNodeType& reference; + typedef const GraphNodeType& const_reference; + typedef std::forward_iterator_tag iterator_category; + + //! Copy constructor + graph_iterator(const graph_iterator& other) : + my_graph(other.my_graph), current_node(other.current_node) + {} + + //! Assignment + graph_iterator& operator=(const graph_iterator& other) { + if (this != &other) { + my_graph = other.my_graph; + current_node = other.current_node; + } + return *this; + } + + //! Dereference + reference operator*() const; + + //! Dereference + pointer operator->() const; + + //! Equality + bool operator==(const graph_iterator& other) const { + return ((my_graph == other.my_graph) && (current_node == other.current_node)); + } + +#if !__TBB_CPP20_COMPARISONS_PRESENT + //! Inequality + bool operator!=(const graph_iterator& other) const { return !(operator==(other)); } +#endif + + //! Pre-increment + graph_iterator& operator++() { + internal_forward(); + return *this; + } + + //! Post-increment + graph_iterator operator++(int) { + graph_iterator result = *this; + operator++(); + return result; + } + +private: + // the graph over which we are iterating + GraphContainerType *my_graph; + // pointer into my_graph's my_nodes list + pointer current_node; + + //! Private initializing constructor for begin() and end() iterators + graph_iterator(GraphContainerType *g, bool begin); + void internal_forward(); +}; // class graph_iterator + +// flags to modify the behavior of the graph reset(). Can be combined. +enum reset_flags { + rf_reset_protocol = 0, + rf_reset_bodies = 1 << 0, // delete the current node body, reset to a copy of the initial node body. + rf_clear_edges = 1 << 1 // delete edges +}; + +void activate_graph(graph& g); +void deactivate_graph(graph& g); +bool is_graph_active(graph& g); +graph_task* prioritize_task(graph& g, graph_task& arena_task); +void spawn_in_graph_arena(graph& g, graph_task& arena_task); +void enqueue_in_graph_arena(graph &g, graph_task& arena_task); + +class graph; + +//! Base class for tasks generated by graph nodes. +class graph_task : public d1::task { +public: + graph_task(graph& g, d1::small_object_allocator& allocator, + node_priority_t node_priority = no_priority); + + graph& my_graph; // graph instance the task belongs to + // TODO revamp: rename to my_priority + node_priority_t priority; + template + void destruct_and_deallocate(const d1::execution_data& ed); +protected: + template + void finalize(const d1::execution_data& ed); +private: + // To organize task_list + graph_task* my_next{ nullptr }; + d1::small_object_allocator my_allocator; + d1::wait_tree_vertex_interface* my_reference_vertex; + // TODO revamp: elaborate internal interfaces to avoid friends declarations + friend class graph_task_list; + friend graph_task* prioritize_task(graph& g, graph_task& gt); +}; + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +class trackable_messages_graph_task : public graph_task { +public: + trackable_messages_graph_task(graph& g, d1::small_object_allocator& allocator, + node_priority_t node_priority, + const std::forward_list& msg_waiters) + : graph_task(g, allocator, node_priority) + , my_msg_wait_context_vertices(msg_waiters) + { + auto last_iterator = my_msg_reference_vertices.cbefore_begin(); + + for (auto& msg_waiter : my_msg_wait_context_vertices) { + d1::wait_tree_vertex_interface* ref_vertex = r1::get_thread_reference_vertex(msg_waiter); + last_iterator = my_msg_reference_vertices.emplace_after(last_iterator, + ref_vertex); + ref_vertex->reserve(1); + } + } + + trackable_messages_graph_task(graph& g, d1::small_object_allocator& allocator, + node_priority_t node_priority, + std::forward_list&& msg_waiters) + : graph_task(g, allocator, node_priority) + , my_msg_wait_context_vertices(std::move(msg_waiters)) + { + } + + const std::forward_list get_msg_wait_context_vertices() const { + return my_msg_wait_context_vertices; + } + +protected: + template + void finalize(const d1::execution_data& ed) { + auto wait_context_vertices = std::move(my_msg_wait_context_vertices); + auto msg_reference_vertices = std::move(my_msg_reference_vertices); + graph_task::finalize(ed); + + // If there is no thread reference vertices associated with the task + // then this task was created by transferring the ownership from other metainfo + // instance (e.g. while taking from the buffer) + if (msg_reference_vertices.empty()) { + for (auto& msg_waiter : wait_context_vertices) { + msg_waiter->release(1); + } + } else { + for (auto& msg_waiter : msg_reference_vertices) { + msg_waiter->release(1); + } + } + } +private: + // Each task that holds information about single message wait_contexts should hold two lists + // The first one is wait_contexts associated with the message itself. They are needed + // to be able to broadcast the list of wait_contexts to the node successors while executing the task. + // The second list is a list of reference vertices for each wait_context_vertex in the first list + // to support the distributed reference counting schema + std::forward_list my_msg_wait_context_vertices; + std::forward_list my_msg_reference_vertices; +}; // class trackable_messages_graph_task +#endif // __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + +struct graph_task_comparator { + bool operator()(const graph_task* left, const graph_task* right) { + return left->priority < right->priority; + } +}; + +typedef tbb::concurrent_priority_queue graph_task_priority_queue_t; + +class priority_task_selector : public d1::task { +public: + priority_task_selector(graph_task_priority_queue_t& priority_queue, d1::small_object_allocator& allocator) + : my_priority_queue(priority_queue), my_allocator(allocator), my_task() {} + task* execute(d1::execution_data& ed) override { + next_task(); + __TBB_ASSERT(my_task, nullptr); + task* t_next = my_task->execute(ed); + my_allocator.delete_object(this, ed); + return t_next; + } + task* cancel(d1::execution_data& ed) override { + if (!my_task) { + next_task(); + } + __TBB_ASSERT(my_task, nullptr); + task* t_next = my_task->cancel(ed); + my_allocator.delete_object(this, ed); + return t_next; + } +private: + void next_task() { + // TODO revamp: hold functors in priority queue instead of real tasks + bool result = my_priority_queue.try_pop(my_task); + __TBB_ASSERT_EX(result, "Number of critical tasks for scheduler and tasks" + " in graph's priority queue mismatched"); + __TBB_ASSERT(my_task && my_task != SUCCESSFULLY_ENQUEUED, + "Incorrect task submitted to graph priority queue"); + __TBB_ASSERT(my_task->priority != no_priority, + "Tasks from graph's priority queue must have priority"); + } + + graph_task_priority_queue_t& my_priority_queue; + d1::small_object_allocator my_allocator; + graph_task* my_task; +}; + +template class run_and_put_task; +template class run_task; + +//******************************************************************************** +// graph tasks helpers +//******************************************************************************** + +//! The list of graph tasks +class graph_task_list : no_copy { +private: + graph_task* my_first; + graph_task** my_next_ptr; +public: + //! Construct empty list + graph_task_list() : my_first(nullptr), my_next_ptr(&my_first) {} + + //! True if list is empty; false otherwise. + bool empty() const { return !my_first; } + + //! Push task onto back of list. + void push_back(graph_task& task) { + task.my_next = nullptr; + *my_next_ptr = &task; + my_next_ptr = &task.my_next; + } + + //! Pop the front task from the list. + graph_task& pop_front() { + __TBB_ASSERT(!empty(), "attempt to pop item from empty task_list"); + graph_task* result = my_first; + my_first = result->my_next; + if (!my_first) { + my_next_ptr = &my_first; + } + return *result; + } +}; + +//! The graph class +/** This class serves as a handle to the graph */ +class graph : no_copy, public graph_proxy { + friend class graph_node; + + void prepare_task_arena(bool reinit = false) { + if (reinit) { + __TBB_ASSERT(my_task_arena, "task arena is nullptr"); + my_task_arena->terminate(); + my_task_arena->initialize(task_arena::attach()); + } + else { + __TBB_ASSERT(my_task_arena == nullptr, "task arena is not nullptr"); + my_task_arena = new task_arena(task_arena::attach()); + } + if (!my_task_arena->is_active()) // failed to attach + my_task_arena->initialize(); // create a new, default-initialized arena + __TBB_ASSERT(my_task_arena->is_active(), "task arena is not active"); + } + +public: + //! Constructs a graph with isolated task_group_context + graph(); + + //! Constructs a graph with use_this_context as context + explicit graph(task_group_context& use_this_context); + + //! Destroys the graph. + /** Calls wait_for_all, then destroys the root task and context. */ + ~graph(); + + //! Used to register that an external entity may still interact with the graph. + /** The graph will not return from wait_for_all until a matching number of release_wait calls is + made. */ + void reserve_wait() override; + + //! Deregisters an external entity that may have interacted with the graph. + /** The graph will not return from wait_for_all until all the number of reserve_wait calls + matches the number of release_wait calls. */ + void release_wait() override; + + //! Wait until graph is idle and the number of release_wait calls equals to the number of + //! reserve_wait calls. + /** The waiting thread will go off and steal work while it is blocked in the wait_for_all. */ + void wait_for_all() { + cancelled = false; + caught_exception = false; + try_call([this] { + my_task_arena->execute([this] { + d1::wait(my_wait_context_vertex.get_context(), *my_context); + }); + cancelled = my_context->is_group_execution_cancelled(); + }).on_exception([this] { + my_context->reset(); + caught_exception = true; + cancelled = true; + }); + // TODO: the "if" condition below is just a work-around to support the concurrent wait + // mode. The cancellation and exception mechanisms are still broken in this mode. + // Consider using task group not to re-implement the same functionality. + if (!(my_context->traits() & task_group_context::concurrent_wait)) { + my_context->reset(); // consistent with behavior in catch() + } + } + + // TODO revamp: consider adding getter for task_group_context. + + // ITERATORS + template + friend class graph_iterator; + + // Graph iterator typedefs + typedef graph_iterator iterator; + typedef graph_iterator const_iterator; + + // Graph iterator constructors + //! start iterator + iterator begin(); + //! end iterator + iterator end(); + //! start const iterator + const_iterator begin() const; + //! end const iterator + const_iterator end() const; + //! start const iterator + const_iterator cbegin() const; + //! end const iterator + const_iterator cend() const; + + // thread-unsafe state reset. + void reset(reset_flags f = rf_reset_protocol); + + //! cancels execution of the associated task_group_context + void cancel(); + + //! return status of graph execution + bool is_cancelled() { return cancelled; } + bool exception_thrown() { return caught_exception; } + +private: + d1::wait_context_vertex my_wait_context_vertex; + task_group_context *my_context; + bool own_context; + bool cancelled; + bool caught_exception; + bool my_is_active; + + graph_node *my_nodes, *my_nodes_last; + + tbb::spin_mutex nodelist_mutex; + void register_node(graph_node *n); + void remove_node(graph_node *n); + + task_arena* my_task_arena; + + graph_task_priority_queue_t my_priority_queue; + + d1::wait_context_vertex& get_wait_context_vertex() { return my_wait_context_vertex; } + + friend void activate_graph(graph& g); + friend void deactivate_graph(graph& g); + friend bool is_graph_active(graph& g); + friend graph_task* prioritize_task(graph& g, graph_task& arena_task); + friend void spawn_in_graph_arena(graph& g, graph_task& arena_task); + friend void enqueue_in_graph_arena(graph &g, graph_task& arena_task); + + friend class d1::task_arena_base; + friend class graph_task; + + template + friend class receiver; +}; // class graph + +template +inline void graph_task::destruct_and_deallocate(const d1::execution_data& ed) { + auto allocator = my_allocator; + // TODO: investigate if direct call of derived destructor gives any benefits. + this->~graph_task(); + allocator.deallocate(static_cast(this), ed); +} + +template +inline void graph_task::finalize(const d1::execution_data& ed) { + d1::wait_tree_vertex_interface* reference_vertex = my_reference_vertex; + destruct_and_deallocate(ed); + reference_vertex->release(); +} + +inline graph_task::graph_task(graph& g, d1::small_object_allocator& allocator, + node_priority_t node_priority) + : my_graph(g) + , priority(node_priority) + , my_allocator(allocator) +{ + d1::wait_context_vertex* graph_wait_context_vertex = &my_graph.get_wait_context_vertex(); + my_reference_vertex = r1::get_thread_reference_vertex(graph_wait_context_vertex); + __TBB_ASSERT(my_reference_vertex, nullptr); + my_reference_vertex->reserve(); +} + +//******************************************************************************** +// end of graph tasks helpers +//******************************************************************************** + + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET +class get_graph_helper; +#endif + +//! The base of all graph nodes. +class graph_node : no_copy { + friend class graph; + template + friend class graph_iterator; + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + friend class get_graph_helper; +#endif + +protected: + graph& my_graph; + graph& graph_reference() const { + // TODO revamp: propagate graph_reference() method to all the reference places. + return my_graph; + } + graph_node* next = nullptr; + graph_node* prev = nullptr; +public: + explicit graph_node(graph& g); + + virtual ~graph_node(); + +protected: + // performs the reset on an individual node. + virtual void reset_node(reset_flags f = rf_reset_protocol) = 0; +}; // class graph_node + +inline void activate_graph(graph& g) { + g.my_is_active = true; +} + +inline void deactivate_graph(graph& g) { + g.my_is_active = false; +} + +inline bool is_graph_active(graph& g) { + return g.my_is_active; +} + +inline graph_task* prioritize_task(graph& g, graph_task& gt) { + if( no_priority == gt.priority ) + return > + + //! Non-preemptive priority pattern. The original task is submitted as a work item to the + //! priority queue, and a new critical task is created to take and execute a work item with + //! the highest known priority. The reference counting responsibility is transferred to + //! the new task. + // A newly created small_object_allocator should be used to allocate the priority_task_selector + // instead of the allocator, associated with gt since gt can be allocated by another thread + d1::small_object_allocator allocator; + d1::task* critical_task = allocator.new_object(g.my_priority_queue, allocator); + __TBB_ASSERT( critical_task, "bad_alloc?" ); + g.my_priority_queue.push(>); + using tbb::detail::d1::submit; + submit( *critical_task, *g.my_task_arena, *g.my_context, /*as_critical=*/true ); + return nullptr; +} + +//! Spawns a task inside graph arena +inline void spawn_in_graph_arena(graph& g, graph_task& arena_task) { + if (is_graph_active(g)) { + d1::task* gt = prioritize_task(g, arena_task); + if( !gt ) + return; + + __TBB_ASSERT(g.my_task_arena && g.my_task_arena->is_active(), nullptr); + submit( *gt, *g.my_task_arena, *g.my_context +#if __TBB_CRITICAL_TASKS + , /*as_critical=*/false +#endif + ); + } +} + +// TODO revamp: unify *_in_graph_arena functions + +//! Enqueues a task inside graph arena +inline void enqueue_in_graph_arena(graph &g, graph_task& arena_task) { + if (is_graph_active(g)) { + __TBB_ASSERT( g.my_task_arena && g.my_task_arena->is_active(), "Is graph's arena initialized and active?" ); + + // TODO revamp: decide on the approach that does not postpone critical task + if( d1::task* gt = prioritize_task(g, arena_task) ) + submit( *gt, *g.my_task_arena, *g.my_context, /*as_critical=*/false); + } +} + +} // namespace d2 +} // namespace detail +} // namespace tbb + +#endif // __TBB_flow_graph_impl_H diff --git a/src/3rd party/oneapi/tbb/detail/_flow_graph_indexer_impl.h b/src/3rd party/oneapi/tbb/detail/_flow_graph_indexer_impl.h new file mode 100644 index 0000000000..7c9a04ec6a --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_flow_graph_indexer_impl.h @@ -0,0 +1,248 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + Copyright (c) 2025 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__flow_graph_indexer_impl_H +#define __TBB__flow_graph_indexer_impl_H + +#ifndef __TBB_flow_graph_H +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +// included in namespace tbb::detail::d2 + +#include "_flow_graph_types_impl.h" + + // Output of the indexer_node is a tbb::flow::tagged_msg, and will be of + // the form tagged_msg + // where the value of tag will indicate which result was put to the + // successor. + + template + graph_task* do_try_put(const T &v, void *p __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) { + typename IndexerNodeBaseType::output_type o(K, v); + return reinterpret_cast(p)->try_put_task(&o __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + } + + template + struct indexer_helper { + template + static inline void set_indexer_node_pointer(PortTuple &my_input, IndexerNodeBaseType *p, graph& g) { + typedef typename std::tuple_element::type T; + auto indexer_node_put_task = do_try_put; + std::get(my_input).set_up(p, indexer_node_put_task, g); + indexer_helper::template set_indexer_node_pointer(my_input, p, g); + } + }; + + template + struct indexer_helper { + template + static inline void set_indexer_node_pointer(PortTuple &my_input, IndexerNodeBaseType *p, graph& g) { + typedef typename std::tuple_element<0, TupleTypes>::type T; + auto indexer_node_put_task = do_try_put; + std::get<0>(my_input).set_up(p, indexer_node_put_task, g); + } + }; + + template + class indexer_input_port : public receiver { + private: + void* my_indexer_ptr; + typedef graph_task* (* forward_function_ptr)(T const &, void* + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo&)); + forward_function_ptr my_try_put_task; + graph* my_graph; + public: + void set_up(void* p, forward_function_ptr f, graph& g) { + my_indexer_ptr = p; + my_try_put_task = f; + my_graph = &g; + } + + protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + graph_task* try_put_task(const T &v) override { + return my_try_put_task(v, my_indexer_ptr __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const T& v, const message_metainfo& metainfo) override { + return my_try_put_task(v, my_indexer_ptr, metainfo); + } +#endif + + graph& graph_reference() const override { + return *my_graph; + } + }; + + template + class indexer_node_FE { + public: + static const int N = std::tuple_size::value; + typedef OutputType output_type; + typedef InputTuple input_type; + + // Some versions of Intel(R) C++ Compiler fail to generate an implicit constructor for the class which has std::tuple as a member. + indexer_node_FE() : my_inputs() {} + + input_type &input_ports() { return my_inputs; } + protected: + input_type my_inputs; + }; + + //! indexer_node_base + template + class indexer_node_base : public graph_node, public indexer_node_FE, + public sender { + protected: + using graph_node::my_graph; + public: + static const size_t N = std::tuple_size::value; + typedef OutputType output_type; + typedef StructTypes tuple_types; + typedef typename sender::successor_type successor_type; + typedef indexer_node_FE input_ports_type; + + private: + // ----------- Aggregator ------------ + enum op_type { reg_succ, rem_succ, try__put_task + }; + typedef indexer_node_base class_type; + + class indexer_node_base_operation : public d1::aggregated_operation { + public: + char type; + union { + output_type const *my_arg; + successor_type *my_succ; + graph_task* bypass_t; + }; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo const* metainfo; +#endif + indexer_node_base_operation(const output_type* e, op_type t) : + type(char(t)), my_arg(e) __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(nullptr)) + {} +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + indexer_node_base_operation(const output_type* e, op_type t, const message_metainfo& info) + : type(char(t)), my_arg(e), metainfo(&info) {} +#endif + indexer_node_base_operation(const successor_type &s, op_type t) : type(char(t)), + my_succ(const_cast(&s)) {} + }; + + typedef d1::aggregating_functor handler_type; + friend class d1::aggregating_functor; + d1::aggregator my_aggregator; + + void handle_operations(indexer_node_base_operation* op_list) { + indexer_node_base_operation *current; + while(op_list) { + current = op_list; + op_list = op_list->next; + switch(current->type) { + + case reg_succ: + my_successors.register_successor(*(current->my_succ)); + current->status.store( SUCCEEDED, std::memory_order_release); + break; + + case rem_succ: + my_successors.remove_successor(*(current->my_succ)); + current->status.store( SUCCEEDED, std::memory_order_release); + break; + case try__put_task: { + current->bypass_t = my_successors.try_put_task(*(current->my_arg) + __TBB_FLOW_GRAPH_METAINFO_ARG(*(current->metainfo))); + current->status.store( SUCCEEDED, std::memory_order_release); // return of try_put_task actual return value + } + break; + } + } + } + // ---------- end aggregator ----------- + public: + indexer_node_base(graph& g) : graph_node(g), input_ports_type(), my_successors(this) { + indexer_helper::set_indexer_node_pointer(this->my_inputs, this, g); + my_aggregator.initialize_handler(handler_type(this)); + } + + indexer_node_base(const indexer_node_base& other) + : graph_node(other.my_graph), input_ports_type(), sender(), my_successors(this) + { + indexer_helper::set_indexer_node_pointer(this->my_inputs, this, other.my_graph); + my_aggregator.initialize_handler(handler_type(this)); + } + + bool register_successor(successor_type &r) override { + indexer_node_base_operation op_data(r, reg_succ); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + + bool remove_successor( successor_type &r) override { + indexer_node_base_operation op_data(r, rem_succ); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + + // not a virtual method in this class + graph_task* try_put_task(output_type const *v + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + { + indexer_node_base_operation op_data(v, try__put_task __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + my_aggregator.execute(&op_data); + return op_data.bypass_t; + } + + protected: + void reset_node(reset_flags f) override { + if(f & rf_clear_edges) { + my_successors.clear(); + } + } + + private: + broadcast_cache my_successors; + }; //indexer_node_base + + // type generators + template + struct indexer_types { + using output_type = tagged_msg; + using input_ports_type = std::tuple...>; + using indexer_base_type = indexer_node_base>; + }; + + template + class unfolded_indexer_node : public indexer_types::indexer_base_type { + public: + using input_ports_type = typename indexer_types::input_ports_type; + using output_type = typename indexer_types::output_type; + using tuple_types = std::tuple; + + private: + using base_type = typename indexer_types::indexer_base_type; + public: + unfolded_indexer_node(graph& g) : base_type(g) {} + unfolded_indexer_node(const unfolded_indexer_node &other) : base_type(other) {} + }; + +#endif /* __TBB__flow_graph_indexer_impl_H */ diff --git a/src/3rd party/oneapi/tbb/detail/_flow_graph_item_buffer_impl.h b/src/3rd party/oneapi/tbb/detail/_flow_graph_item_buffer_impl.h new file mode 100644 index 0000000000..cf7c54b852 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_flow_graph_item_buffer_impl.h @@ -0,0 +1,422 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__flow_graph_item_buffer_impl_H +#define __TBB__flow_graph_item_buffer_impl_H + +#ifndef __TBB_flow_graph_H +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +#include "_aligned_space.h" + +// in namespace tbb::flow::interfaceX (included in _flow_graph_node_impl.h) + +//! Expandable buffer of items. The possible operations are push, pop, +//* tests for empty and so forth. No mutual exclusion is built in. +//* objects are constructed into and explicitly-destroyed. get_my_item gives +// a read-only reference to the item in the buffer. set_my_item may be called +// with either an empty or occupied slot. + +template > +class item_buffer { +public: + typedef T item_type; + enum buffer_item_state { no_item=0, has_item=1, reserved_item=2 }; +protected: + struct aligned_space_item { + item_type item; + buffer_item_state state; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo metainfo; +#endif + }; + typedef size_t size_type; + typedef aligned_space buffer_item_type; + typedef typename allocator_traits::template rebind_alloc allocator_type; + buffer_item_type *my_array; + size_type my_array_size; + static const size_type initial_buffer_size = 4; + size_type my_head; + size_type my_tail; + + bool buffer_empty() const { return my_head == my_tail; } + + aligned_space_item &element(size_type i) { + __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->state))%alignment_of::value), nullptr); + __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->item))%alignment_of::value), nullptr); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->metainfo))%alignment_of::value), nullptr); +#endif + return *my_array[i & (my_array_size - 1) ].begin(); + } + + const aligned_space_item &element(size_type i) const { + __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->state))%alignment_of::value), nullptr); + __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->item))%alignment_of::value), nullptr); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].begin()->metainfo))%alignment_of::value), nullptr); +#endif + return *my_array[i & (my_array_size-1)].begin(); + } + + bool my_item_valid(size_type i) const { return (i < my_tail) && (i >= my_head) && (element(i).state != no_item); } +#if TBB_USE_ASSERT + bool my_item_reserved(size_type i) const { return element(i).state == reserved_item; } +#endif + + // object management in buffer + const item_type &get_my_item(size_t i) const { + __TBB_ASSERT(my_item_valid(i),"attempt to get invalid item"); + return element(i).item; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo& get_my_metainfo(size_t i) { + __TBB_ASSERT(my_item_valid(i), "attempt to get invalid item"); + return element(i).metainfo; + } +#endif + + // may be called with an empty slot or a slot that has already been constructed into. + void set_my_item(size_t i, const item_type &o + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + { + if(element(i).state != no_item) { + destroy_item(i); + } + new(&(element(i).item)) item_type(o); + element(i).state = has_item; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + new(&element(i).metainfo) message_metainfo(metainfo); + + for (auto& waiter : metainfo.waiters()) { + waiter->reserve(1); + } +#endif + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + void set_my_item(size_t i, const item_type& o, message_metainfo&& metainfo) { + if(element(i).state != no_item) { + destroy_item(i); + } + + new(&(element(i).item)) item_type(o); + new(&element(i).metainfo) message_metainfo(std::move(metainfo)); + // Skipping the reservation on metainfo.waiters since the ownership + // is moving from metainfo to the cache + element(i).state = has_item; + } +#endif + + // destructively-fetch an object from the buffer +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + void fetch_item(size_t i, item_type& o, message_metainfo& metainfo) { + __TBB_ASSERT(my_item_valid(i), "Trying to fetch an empty slot"); + o = get_my_item(i); // could have std::move assign semantics + metainfo = std::move(get_my_metainfo(i)); + destroy_item(i); + } +#else + void fetch_item(size_t i, item_type &o) { + __TBB_ASSERT(my_item_valid(i), "Trying to fetch an empty slot"); + o = get_my_item(i); // could have std::move assign semantics + destroy_item(i); + } +#endif + + // move an existing item from one slot to another. The moved-to slot must be unoccupied, + // the moved-from slot must exist and not be reserved. The after, from will be empty, + // to will be occupied but not reserved + void move_item(size_t to, size_t from) { + __TBB_ASSERT(!my_item_valid(to), "Trying to move to a non-empty slot"); + __TBB_ASSERT(my_item_valid(from), "Trying to move from an empty slot"); + // could have std::move semantics + set_my_item(to, get_my_item(from) __TBB_FLOW_GRAPH_METAINFO_ARG(get_my_metainfo(from))); + destroy_item(from); + } + + // put an item in an empty slot. Return true if successful, else false +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + bool place_item(size_t here, const item_type &me, Metainfo&& metainfo) { +#if !TBB_DEPRECATED_SEQUENCER_DUPLICATES + if(my_item_valid(here)) return false; +#endif + set_my_item(here, me, std::forward(metainfo)); + return true; + } +#else + bool place_item(size_t here, const item_type &me) { +#if !TBB_DEPRECATED_SEQUENCER_DUPLICATES + if(my_item_valid(here)) return false; +#endif + set_my_item(here, me); + return true; + } +#endif + + // could be implemented with std::move semantics + void swap_items(size_t i, size_t j) { + __TBB_ASSERT(my_item_valid(i) && my_item_valid(j), "attempt to swap invalid item(s)"); + item_type temp = get_my_item(i); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo temp_metainfo = get_my_metainfo(i); + set_my_item(i, get_my_item(j), get_my_metainfo(j)); + set_my_item(j, temp, temp_metainfo); +#else + set_my_item(i, get_my_item(j)); + set_my_item(j, temp); +#endif + } + + void destroy_item(size_type i) { + __TBB_ASSERT(my_item_valid(i), "destruction of invalid item"); + + auto& e = element(i); + e.item.~item_type(); + e.state = no_item; + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + for (auto& msg_waiter : e.metainfo.waiters()) { + msg_waiter->release(1); + } + + e.metainfo.~message_metainfo(); +#endif + } + + // returns the front element + const item_type& front() const + { + __TBB_ASSERT(my_item_valid(my_head), "attempt to fetch head non-item"); + return get_my_item(my_head); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + const message_metainfo& front_metainfo() const + { + __TBB_ASSERT(my_item_valid(my_head), "attempt to fetch head non-item"); + return element(my_head).metainfo; + } +#endif + + // returns the back element + const item_type& back() const + { + __TBB_ASSERT(my_item_valid(my_tail - 1), "attempt to fetch head non-item"); + return get_my_item(my_tail - 1); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + const message_metainfo& back_metainfo() const { + __TBB_ASSERT(my_item_valid(my_tail - 1), "attempt to fetch head non-item"); + return element(my_tail - 1).metainfo; + } +#endif + + // following methods are for reservation of the front of a buffer. + void reserve_item(size_type i) { + __TBB_ASSERT(my_item_valid(i) && !my_item_reserved(i), "item cannot be reserved"); + element(i).state = reserved_item; + } + + void release_item(size_type i) { + __TBB_ASSERT(my_item_reserved(i), "item is not reserved"); + element(i).state = has_item; + } + + void destroy_front() { destroy_item(my_head); ++my_head; } + void destroy_back() { destroy_item(my_tail-1); --my_tail; } + + // we have to be able to test against a new tail value without changing my_tail + // grow_array doesn't work if we change my_tail when the old array is too small + size_type size(size_t new_tail = 0) { return (new_tail ? new_tail : my_tail) - my_head; } + size_type capacity() { return my_array_size; } + // sequencer_node does not use this method, so we don't + // need a version that passes in the new_tail value. + bool buffer_full() { return size() >= capacity(); } + + //! Grows the internal array. + void grow_my_array( size_t minimum_size ) { + // test that we haven't made the structure inconsistent. + __TBB_ASSERT(capacity() >= my_tail - my_head, "total items exceed capacity"); + size_type new_size = my_array_size ? 2*my_array_size : initial_buffer_size; + while( new_sizestate = no_item; } + + for( size_type i=my_head; iitem); + (void)new(new_space) item_type(get_my_item(i)); + new_array[i&(new_size-1)].begin()->state = element(i).state; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + char* meta_space = (char *)&(new_array[i&(new_size-1)].begin()->metainfo); + ::new(meta_space) message_metainfo(std::move(element(i).metainfo)); +#endif + } + } + + clean_up_buffer(/*reset_pointers*/false); + + my_array = new_array; + my_array_size = new_size; + } + + bool push_back(item_type& v + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + { + if (buffer_full()) { + grow_my_array(size() + 1); + } + set_my_item(my_tail, v __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + ++my_tail; + return true; + } + + bool pop_back(item_type& v + __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& metainfo)) + { + if (!my_item_valid(my_tail - 1)) { + return false; + } + auto& e = element(my_tail - 1); + v = e.item; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + metainfo = std::move(e.metainfo); +#endif + + destroy_back(); + return true; + } + + bool pop_front(item_type& v + __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& metainfo)) + { + if (!my_item_valid(my_head)) { + return false; + } + auto& e = element(my_head); + v = e.item; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + metainfo = std::move(e.metainfo); +#endif + + destroy_front(); + return true; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool pop_back(item_type& v) { + message_metainfo metainfo; + return pop_back(v, metainfo); + } + + bool pop_front(item_type& v) { + message_metainfo metainfo; + return pop_front(v, metainfo); + } +#endif + + // This is used both for reset and for grow_my_array. In the case of grow_my_array + // we want to retain the values of the head and tail. + void clean_up_buffer(bool reset_pointers) { + if (my_array) { + for( size_type i=my_head; i > +class reservable_item_buffer : public item_buffer { +protected: + using item_buffer::my_item_valid; + using item_buffer::my_head; + +public: + reservable_item_buffer() : item_buffer(), my_reserved(false) {} + void reset() {my_reserved = false; item_buffer::reset(); } +protected: + + bool reserve_front(T &v) { + if(my_reserved || !my_item_valid(this->my_head)) return false; + my_reserved = true; + // reserving the head + v = this->front(); + this->reserve_item(this->my_head); + return true; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool reserve_front(T& v, message_metainfo& metainfo) { + if (my_reserved || !my_item_valid(this->my_head)) return false; + my_reserved = true; + // reserving the head + v = this->front(); + metainfo = this->front_metainfo(); + this->reserve_item(this->my_head); + return true; + } +#endif + + void consume_front() { + __TBB_ASSERT(my_reserved, "Attempt to consume a non-reserved item"); + this->destroy_front(); + my_reserved = false; + } + + void release_front() { + __TBB_ASSERT(my_reserved, "Attempt to release a non-reserved item"); + this->release_item(this->my_head); + my_reserved = false; + } + + bool my_reserved; +}; + +#endif // __TBB__flow_graph_item_buffer_impl_H diff --git a/src/3rd party/oneapi/tbb/detail/_flow_graph_join_impl.h b/src/3rd party/oneapi/tbb/detail/_flow_graph_join_impl.h new file mode 100644 index 0000000000..706df447a9 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_flow_graph_join_impl.h @@ -0,0 +1,1544 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + Copyright (c) 2025 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__flow_graph_join_impl_H +#define __TBB__flow_graph_join_impl_H + +#ifndef __TBB_flow_graph_H +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +// included into namespace tbb::detail::d2 + + struct forwarding_base : no_assign { + forwarding_base(graph &g) : graph_ref(g) {} + virtual ~forwarding_base() {} + graph& graph_ref; + }; + + struct queueing_forwarding_base : forwarding_base { + using forwarding_base::forwarding_base; + // decrement_port_count may create a forwarding task. If we cannot handle the task + // ourselves, ask decrement_port_count to deal with it. + virtual graph_task* decrement_port_count(bool handle_task) = 0; + }; + + struct reserving_forwarding_base : forwarding_base { + using forwarding_base::forwarding_base; + // decrement_port_count may create a forwarding task. If we cannot handle the task + // ourselves, ask decrement_port_count to deal with it. + virtual graph_task* decrement_port_count() = 0; + virtual void increment_port_count() = 0; + }; + + // specialization that lets us keep a copy of the current_key for building results. + // KeyType can be a reference type. + template + struct matching_forwarding_base : public forwarding_base { + typedef typename std::decay::type current_key_type; + matching_forwarding_base(graph &g) : forwarding_base(g) { } + virtual graph_task* increment_key_count(current_key_type const & /*t*/) = 0; + current_key_type current_key; // so ports can refer to FE's desired items + }; + + template< int N > + struct join_helper { + + template< typename TupleType, typename PortType > + static inline void set_join_node_pointer(TupleType &my_input, PortType *port) { + std::get( my_input ).set_join_node_pointer(port); + join_helper::set_join_node_pointer( my_input, port ); + } + template< typename TupleType > + static inline void consume_reservations( TupleType &my_input ) { + std::get( my_input ).consume(); + join_helper::consume_reservations( my_input ); + } + + template< typename TupleType > + static inline void release_my_reservation( TupleType &my_input ) { + std::get( my_input ).release(); + } + + template + static inline void release_reservations( TupleType &my_input) { + join_helper::release_reservations(my_input); + release_my_reservation(my_input); + } + + template< typename InputTuple, typename OutputTuple > + static inline bool reserve( InputTuple &my_input, OutputTuple &out) { + if ( !std::get( my_input ).reserve( std::get( out ) ) ) return false; + if ( !join_helper::reserve( my_input, out ) ) { + release_my_reservation( my_input ); + return false; + } + return true; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + static inline bool reserve(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) { + message_metainfo element_metainfo; + if (!std::get(my_input).reserve(std::get(out), element_metainfo)) return false; + if (!join_helper::reserve(my_input, out, metainfo)) { + release_my_reservation(my_input); + return false; + } + metainfo.merge(element_metainfo); + return true; + + } +#endif + + template + static inline bool get_my_item( InputTuple &my_input, OutputTuple &out) { + bool res = std::get(my_input).get_item(std::get(out) ); // may fail + return join_helper::get_my_item(my_input, out) && res; // do get on other inputs before returning + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + static inline bool get_my_item(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) { + message_metainfo element_metainfo; + bool res = std::get(my_input).get_item(std::get(out), element_metainfo); + metainfo.merge(element_metainfo); + return join_helper::get_my_item(my_input, out, metainfo) && res; + } +#endif + + template + static inline bool get_items(InputTuple &my_input, OutputTuple &out) { + return get_my_item(my_input, out); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + static inline bool get_items(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) { + return get_my_item(my_input, out, metainfo); + } +#endif + + template + static inline void reset_my_port(InputTuple &my_input) { + join_helper::reset_my_port(my_input); + std::get(my_input).reset_port(); + } + + template + static inline void reset_ports(InputTuple& my_input) { + reset_my_port(my_input); + } + + template + static inline void set_key_functors(InputTuple &my_input, KeyFuncTuple &my_key_funcs) { + std::get(my_input).set_my_key_func(std::get(my_key_funcs)); + std::get(my_key_funcs) = nullptr; + join_helper::set_key_functors(my_input, my_key_funcs); + } + + template< typename KeyFuncTuple> + static inline void copy_key_functors(KeyFuncTuple &my_inputs, KeyFuncTuple &other_inputs) { + __TBB_ASSERT( + std::get(other_inputs).get_my_key_func(), + "key matching join node should not be instantiated without functors." + ); + std::get(my_inputs).set_my_key_func(std::get(other_inputs).get_my_key_func()->clone()); + join_helper::copy_key_functors(my_inputs, other_inputs); + } + + template + static inline void reset_inputs(InputTuple &my_input, reset_flags f) { + join_helper::reset_inputs(my_input, f); + std::get(my_input).reset_receiver(f); + } + }; // join_helper + + template< > + struct join_helper<1> { + + template< typename TupleType, typename PortType > + static inline void set_join_node_pointer(TupleType &my_input, PortType *port) { + std::get<0>( my_input ).set_join_node_pointer(port); + } + + template< typename TupleType > + static inline void consume_reservations( TupleType &my_input ) { + std::get<0>( my_input ).consume(); + } + + template< typename TupleType > + static inline void release_my_reservation( TupleType &my_input ) { + std::get<0>( my_input ).release(); + } + + template + static inline void release_reservations( TupleType &my_input) { + release_my_reservation(my_input); + } + + template< typename InputTuple, typename OutputTuple > + static inline bool reserve( InputTuple &my_input, OutputTuple &out) { + return std::get<0>( my_input ).reserve( std::get<0>( out ) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + static inline bool reserve(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) { + message_metainfo element_metainfo; + bool result = std::get<0>(my_input).reserve(std::get<0>(out), element_metainfo); + metainfo.merge(element_metainfo); + return result; + } +#endif + + template + static inline bool get_my_item( InputTuple &my_input, OutputTuple &out) { + return std::get<0>(my_input).get_item(std::get<0>(out)); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + static inline bool get_my_item(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) { + message_metainfo element_metainfo; + bool res = std::get<0>(my_input).get_item(std::get<0>(out), element_metainfo); + metainfo.merge(element_metainfo); + return res; + } +#endif + + template + static inline bool get_items(InputTuple &my_input, OutputTuple &out) { + return get_my_item(my_input, out); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + static inline bool get_items(InputTuple& my_input, OutputTuple& out, message_metainfo& metainfo) { + return get_my_item(my_input, out, metainfo); + } +#endif + + template + static inline void reset_my_port(InputTuple &my_input) { + std::get<0>(my_input).reset_port(); + } + + template + static inline void reset_ports(InputTuple& my_input) { + reset_my_port(my_input); + } + + template + static inline void set_key_functors(InputTuple &my_input, KeyFuncTuple &my_key_funcs) { + std::get<0>(my_input).set_my_key_func(std::get<0>(my_key_funcs)); + std::get<0>(my_key_funcs) = nullptr; + } + + template< typename KeyFuncTuple> + static inline void copy_key_functors(KeyFuncTuple &my_inputs, KeyFuncTuple &other_inputs) { + __TBB_ASSERT( + std::get<0>(other_inputs).get_my_key_func(), + "key matching join node should not be instantiated without functors." + ); + std::get<0>(my_inputs).set_my_key_func(std::get<0>(other_inputs).get_my_key_func()->clone()); + } + template + static inline void reset_inputs(InputTuple &my_input, reset_flags f) { + std::get<0>(my_input).reset_receiver(f); + } + }; // join_helper<1> + + //! The two-phase join port + template< typename T > + class reserving_port : public receiver { + public: + typedef T input_type; + typedef typename receiver::predecessor_type predecessor_type; + + private: + // ----------- Aggregator ------------ + enum op_type { reg_pred, rem_pred, res_item, rel_res, con_res + }; + typedef reserving_port class_type; + + class reserving_port_operation : public d1::aggregated_operation { + public: + char type; + union { + T *my_arg; + predecessor_type *my_pred; + }; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo* metainfo; +#endif + reserving_port_operation(const T& e, op_type t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& info)) : + type(char(t)), my_arg(const_cast(&e)) + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(&info)) {} +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + reserving_port_operation(const T& e, op_type t) + : type(char(t)), my_arg(const_cast(&e)), metainfo(nullptr) {} +#endif + reserving_port_operation(const predecessor_type &s, op_type t) : type(char(t)), + my_pred(const_cast(&s)) {} + reserving_port_operation(op_type t) : type(char(t)) {} + }; + + typedef d1::aggregating_functor handler_type; + friend class d1::aggregating_functor; + d1::aggregator my_aggregator; + + void handle_operations(reserving_port_operation* op_list) { + reserving_port_operation *current; + bool was_missing_predecessors = false; + while(op_list) { + current = op_list; + op_list = op_list->next; + switch(current->type) { + case reg_pred: + was_missing_predecessors = my_predecessors.empty(); + my_predecessors.add(*(current->my_pred)); + if ( was_missing_predecessors ) { + (void) my_join->decrement_port_count(); // may try to forward + } + current->status.store( SUCCEEDED, std::memory_order_release); + break; + case rem_pred: + if ( !my_predecessors.empty() ) { + my_predecessors.remove(*(current->my_pred)); + if ( my_predecessors.empty() ) // was the last predecessor + my_join->increment_port_count(); + } + // TODO: consider returning failure if there were no predecessors to remove + current->status.store( SUCCEEDED, std::memory_order_release ); + break; + case res_item: + if ( reserved ) { + current->status.store( FAILED, std::memory_order_release); + } + else { + bool reserve_result = false; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (current->metainfo) { + reserve_result = my_predecessors.try_reserve(*(current->my_arg), + *(current->metainfo)); + } else +#endif + { + reserve_result = my_predecessors.try_reserve(*(current->my_arg)); + } + if (reserve_result) { + reserved = true; + current->status.store( SUCCEEDED, std::memory_order_release); + } else { + if ( my_predecessors.empty() ) { + my_join->increment_port_count(); + } + current->status.store( FAILED, std::memory_order_release); + } + } + break; + case rel_res: + reserved = false; + my_predecessors.try_release( ); + current->status.store( SUCCEEDED, std::memory_order_release); + break; + case con_res: + reserved = false; + my_predecessors.try_consume( ); + current->status.store( SUCCEEDED, std::memory_order_release); + break; + } + } + } + + protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + graph_task* try_put_task( const T & ) override { + return nullptr; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const T&, const message_metainfo&) override { return nullptr; } +#endif + + graph& graph_reference() const override { + return my_join->graph_ref; + } + + public: + + //! Constructor + reserving_port() : my_join(nullptr), my_predecessors(this), reserved(false) { + my_aggregator.initialize_handler(handler_type(this)); + } + + // copy constructor + reserving_port(const reserving_port& /* other */) = delete; + + void set_join_node_pointer(reserving_forwarding_base *join) { + my_join = join; + } + + //! Add a predecessor + bool register_predecessor( predecessor_type &src ) override { + reserving_port_operation op_data(src, reg_pred); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + + //! Remove a predecessor + bool remove_predecessor( predecessor_type &src ) override { + reserving_port_operation op_data(src, rem_pred); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + + //! Reserve an item from the port + bool reserve( T &v ) { + reserving_port_operation op_data(v, res_item); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool reserve( T& v, message_metainfo& metainfo ) { + reserving_port_operation op_data(v, res_item, metainfo); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } +#endif + + //! Release the port + void release( ) { + reserving_port_operation op_data(rel_res); + my_aggregator.execute(&op_data); + } + + //! Complete use of the port + void consume( ) { + reserving_port_operation op_data(con_res); + my_aggregator.execute(&op_data); + } + + void reset_receiver( reset_flags f) { + if(f & rf_clear_edges) my_predecessors.clear(); + else + my_predecessors.reset(); + reserved = false; + __TBB_ASSERT(!(f&rf_clear_edges) || my_predecessors.empty(), "port edges not removed"); + } + + private: +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + friend class get_graph_helper; +#endif + + reserving_forwarding_base *my_join; + reservable_predecessor_cache< T, null_mutex > my_predecessors; + bool reserved; + }; // reserving_port + + //! queueing join_port + template + class queueing_port : public receiver, public item_buffer { + public: + typedef T input_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef queueing_port class_type; + + // ----------- Aggregator ------------ + private: + enum op_type { get__item, res_port, try__put_task + }; + + class queueing_port_operation : public d1::aggregated_operation { + public: + char type; + T my_val; + T* my_arg; + graph_task* bypass_t; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo* metainfo; +#endif + // constructor for value parameter + queueing_port_operation(const T& e, op_type t __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& info)) + : type(char(t)), my_val(e), my_arg(nullptr) + , bypass_t(nullptr) + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(const_cast(&info))) + {} + // constructor for pointer parameter + queueing_port_operation(const T* p, op_type t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& info)) : + type(char(t)), my_arg(const_cast(p)) + , bypass_t(nullptr) + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(&info)) + {} +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + queueing_port_operation(const T* p, op_type t) + : type(char(t)), my_arg(const_cast(p)), bypass_t(nullptr), metainfo(nullptr) + {} +#endif + // constructor with no parameter + queueing_port_operation(op_type t) : type(char(t)), my_arg(nullptr) + , bypass_t(nullptr) + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(nullptr)) + {} + }; + + typedef d1::aggregating_functor handler_type; + friend class d1::aggregating_functor; + d1::aggregator my_aggregator; + + void handle_operations(queueing_port_operation* op_list) { + queueing_port_operation *current; + bool was_empty; + while(op_list) { + current = op_list; + op_list = op_list->next; + switch(current->type) { + case try__put_task: { + graph_task* rtask = nullptr; + was_empty = this->buffer_empty(); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + __TBB_ASSERT(current->metainfo, nullptr); + this->push_back(current->my_val, *(current->metainfo)); +#else + this->push_back(current->my_val); +#endif + if (was_empty) rtask = my_join->decrement_port_count(false); + else + rtask = SUCCESSFULLY_ENQUEUED; + current->bypass_t = rtask; + current->status.store( SUCCEEDED, std::memory_order_release); + } + break; + case get__item: + if(!this->buffer_empty()) { + __TBB_ASSERT(current->my_arg, nullptr); + *(current->my_arg) = this->front(); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (current->metainfo) { + *(current->metainfo) = this->front_metainfo(); + } +#endif + current->status.store( SUCCEEDED, std::memory_order_release); + } + else { + current->status.store( FAILED, std::memory_order_release); + } + break; + case res_port: + __TBB_ASSERT(this->my_item_valid(this->my_head), "No item to reset"); + this->destroy_front(); + if(this->my_item_valid(this->my_head)) { + (void)my_join->decrement_port_count(true); + } + current->status.store( SUCCEEDED, std::memory_order_release); + break; + } + } + } + // ------------ End Aggregator --------------- + + protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + + private: + graph_task* try_put_task_impl(const T& v __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) { + queueing_port_operation op_data(v, try__put_task __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + my_aggregator.execute(&op_data); + __TBB_ASSERT(op_data.status == SUCCEEDED || !op_data.bypass_t, "inconsistent return from aggregator"); + if(!op_data.bypass_t) return SUCCESSFULLY_ENQUEUED; + return op_data.bypass_t; + } + + protected: + graph_task* try_put_task(const T &v) override { + return try_put_task_impl(v __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const T& v, const message_metainfo& metainfo) override { + return try_put_task_impl(v, metainfo); + } +#endif + + graph& graph_reference() const override { + return my_join->graph_ref; + } + + public: + + //! Constructor + queueing_port() : item_buffer() { + my_join = nullptr; + my_aggregator.initialize_handler(handler_type(this)); + } + + //! copy constructor + queueing_port(const queueing_port& /* other */) = delete; + + //! record parent for tallying available items + void set_join_node_pointer(queueing_forwarding_base *join) { + my_join = join; + } + + bool get_item( T &v ) { + queueing_port_operation op_data(&v, get__item); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool get_item( T& v, message_metainfo& metainfo ) { + queueing_port_operation op_data(&v, get__item, metainfo); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } +#endif + + // reset_port is called when item is accepted by successor, but + // is initiated by join_node. + void reset_port() { + queueing_port_operation op_data(res_port); + my_aggregator.execute(&op_data); + return; + } + + void reset_receiver(reset_flags) { + item_buffer::reset(); + } + + private: +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + friend class get_graph_helper; +#endif + + queueing_forwarding_base *my_join; + }; // queueing_port + +#include "_flow_graph_tagged_buffer_impl.h" + + template + struct count_element { + K my_key; + size_t my_value; + }; + + // method to access the key in the counting table + // the ref has already been removed from K + template< typename K > + struct key_to_count_functor { + typedef count_element table_item_type; + const K& operator()(const table_item_type& v) { return v.my_key; } + }; + + template + struct key_matching_port_base { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + using type = metainfo_hash_buffer; +#else + using type = hash_buffer; +#endif + }; + + // the ports can have only one template parameter. We wrap the types needed in + // a traits type + template< class TraitsType > + class key_matching_port : + public receiver, + public key_matching_port_base< typename TraitsType::K, typename TraitsType::T, typename TraitsType::TtoK, + typename TraitsType::KHash >::type + { + public: + typedef TraitsType traits; + typedef key_matching_port class_type; + typedef typename TraitsType::T input_type; + typedef typename TraitsType::K key_type; + typedef typename std::decay::type noref_key_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename TraitsType::TtoK type_to_key_func_type; + typedef typename TraitsType::KHash hash_compare_type; + typedef typename key_matching_port_base::type buffer_type; + + private: +// ----------- Aggregator ------------ + private: + enum op_type { try__put, get__item, res_port + }; + + class key_matching_port_operation : public d1::aggregated_operation { + public: + char type; + input_type my_val; + input_type *my_arg; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo* metainfo = nullptr; +#endif + // constructor for value parameter + key_matching_port_operation(const input_type& e, op_type t + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& info)) + : type(char(t)), my_val(e), my_arg(nullptr) + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(const_cast(&info))) {} + + // constructor for pointer parameter + key_matching_port_operation(const input_type* p, op_type t + __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& info)) + : type(char(t)), my_arg(const_cast(p)) + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(&info)) {} + + // constructor with no parameter + key_matching_port_operation(op_type t) : type(char(t)), my_arg(nullptr) {} + }; + + typedef d1::aggregating_functor handler_type; + friend class d1::aggregating_functor; + d1::aggregator my_aggregator; + + void handle_operations(key_matching_port_operation* op_list) { + key_matching_port_operation *current; + while(op_list) { + current = op_list; + op_list = op_list->next; + switch(current->type) { + case try__put: { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + __TBB_ASSERT(current->metainfo, nullptr); + bool was_inserted = this->insert_with_key(current->my_val, *(current->metainfo)); +#else + bool was_inserted = this->insert_with_key(current->my_val); +#endif + // return failure if a duplicate insertion occurs + current->status.store( was_inserted ? SUCCEEDED : FAILED, std::memory_order_release); + } + break; + case get__item: { + // use current_key from FE for item + __TBB_ASSERT(current->my_arg, nullptr); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + __TBB_ASSERT(current->metainfo, nullptr); + bool find_result = this->find_with_key(my_join->current_key, *(current->my_arg), + *(current->metainfo)); +#else + bool find_result = this->find_with_key(my_join->current_key, *(current->my_arg)); +#endif +#if TBB_USE_DEBUG + if (!find_result) { + __TBB_ASSERT(false, "Failed to find item corresponding to current_key."); + } +#else + tbb::detail::suppress_unused_warning(find_result); +#endif + current->status.store( SUCCEEDED, std::memory_order_release); + } + break; + case res_port: + // use current_key from FE for item + this->delete_with_key(my_join->current_key); + current->status.store( SUCCEEDED, std::memory_order_release); + break; + } + } + } +// ------------ End Aggregator --------------- + protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + private: + graph_task* try_put_task_impl(const input_type& v __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) { + key_matching_port_operation op_data(v, try__put __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + graph_task* rtask = nullptr; + my_aggregator.execute(&op_data); + if(op_data.status == SUCCEEDED) { + rtask = my_join->increment_key_count((*(this->get_key_func()))(v)); // may spawn + // rtask has to reflect the return status of the try_put + if(!rtask) rtask = SUCCESSFULLY_ENQUEUED; + } + return rtask; + } + protected: + graph_task* try_put_task(const input_type& v) override { + return try_put_task_impl(v __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const input_type& v, const message_metainfo& metainfo) override { + return try_put_task_impl(v, metainfo); + } +#endif + + graph& graph_reference() const override { + return my_join->graph_ref; + } + + public: + + key_matching_port() : receiver(), buffer_type() { + my_join = nullptr; + my_aggregator.initialize_handler(handler_type(this)); + } + + // copy constructor + key_matching_port(const key_matching_port& /*other*/) = delete; +#if __INTEL_COMPILER <= 2021 + // Suppress superfluous diagnostic about virtual keyword absence in a destructor of an inherited + // class while the parent class has the virtual keyword for the destrocutor. + virtual +#endif + ~key_matching_port() { } + + void set_join_node_pointer(forwarding_base *join) { + my_join = dynamic_cast*>(join); + } + + void set_my_key_func(type_to_key_func_type *f) { this->set_key_func(f); } + + type_to_key_func_type* get_my_key_func() { return this->get_key_func(); } + + bool get_item( input_type &v ) { + // aggregator uses current_key from FE for Key + key_matching_port_operation op_data(&v, get__item); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool get_item( input_type& v, message_metainfo& metainfo ) { + // aggregator uses current_key from FE for Key + key_matching_port_operation op_data(&v, get__item, metainfo); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } +#endif + + // reset_port is called when item is accepted by successor, but + // is initiated by join_node. + void reset_port() { + key_matching_port_operation op_data(res_port); + my_aggregator.execute(&op_data); + return; + } + + void reset_receiver(reset_flags ) { + buffer_type::reset(); + } + + private: + // my_join forwarding base used to count number of inputs that + // received key. + matching_forwarding_base *my_join; + }; // key_matching_port + + using namespace graph_policy_namespace; + + template + class join_node_base; + + //! join_node_FE : implements input port policy + template + class join_node_FE; + + template + class join_node_FE : public reserving_forwarding_base { + private: + static const int N = std::tuple_size::value; + typedef OutputTuple output_type; + typedef InputTuple input_type; + typedef join_node_base base_node_type; // for forwarding + public: + join_node_FE(graph &g) : reserving_forwarding_base(g), my_node(nullptr) { + ports_with_no_inputs = N; + join_helper::set_join_node_pointer(my_inputs, this); + } + + join_node_FE(const join_node_FE& other) : reserving_forwarding_base((other.reserving_forwarding_base::graph_ref)), my_node(nullptr) { + ports_with_no_inputs = N; + join_helper::set_join_node_pointer(my_inputs, this); + } + + void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; } + + void increment_port_count() override { + ++ports_with_no_inputs; + } + + // if all input_ports have predecessors, spawn forward to try and consume tuples + graph_task* decrement_port_count() override { + if(ports_with_no_inputs.fetch_sub(1) == 1) { + if(is_graph_active(this->graph_ref)) { + d1::small_object_allocator allocator{}; + typedef forward_task_bypass task_type; + graph_task* t = allocator.new_object(graph_ref, allocator, *my_node); + spawn_in_graph_arena(this->graph_ref, *t); + } + } + return nullptr; + } + + input_type &input_ports() { return my_inputs; } + + protected: + + void reset( reset_flags f) { + // called outside of parallel contexts + ports_with_no_inputs = N; + join_helper::reset_inputs(my_inputs, f); + } + + // all methods on input ports should be called under mutual exclusion from join_node_base. + + bool tuple_build_may_succeed() { + return !ports_with_no_inputs; + } + + bool try_to_make_tuple(output_type &out) { + if(ports_with_no_inputs) return false; + return join_helper::reserve(my_inputs, out); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool try_to_make_tuple(output_type &out, message_metainfo& metainfo) { + if (ports_with_no_inputs) return false; + return join_helper::reserve(my_inputs, out, metainfo); + } +#endif + + void tuple_accepted() { + join_helper::consume_reservations(my_inputs); + } + void tuple_rejected() { + join_helper::release_reservations(my_inputs); + } + + input_type my_inputs; + base_node_type *my_node; + std::atomic ports_with_no_inputs; + }; // join_node_FE + + template + class join_node_FE : public queueing_forwarding_base { + public: + static const int N = std::tuple_size::value; + typedef OutputTuple output_type; + typedef InputTuple input_type; + typedef join_node_base base_node_type; // for forwarding + + join_node_FE(graph &g) : queueing_forwarding_base(g), my_node(nullptr) { + ports_with_no_items = N; + join_helper::set_join_node_pointer(my_inputs, this); + } + + join_node_FE(const join_node_FE& other) : queueing_forwarding_base((other.queueing_forwarding_base::graph_ref)), my_node(nullptr) { + ports_with_no_items = N; + join_helper::set_join_node_pointer(my_inputs, this); + } + + // needed for forwarding + void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; } + + void reset_port_count() { + ports_with_no_items = N; + } + + // if all input_ports have items, spawn forward to try and consume tuples + graph_task* decrement_port_count(bool handle_task) override + { + if(ports_with_no_items.fetch_sub(1) == 1) { + if(is_graph_active(this->graph_ref)) { + d1::small_object_allocator allocator{}; + typedef forward_task_bypass task_type; + graph_task* t = allocator.new_object(graph_ref, allocator, *my_node); + if( !handle_task ) + return t; + spawn_in_graph_arena(this->graph_ref, *t); + } + } + return nullptr; + } + + input_type &input_ports() { return my_inputs; } + + protected: + + void reset( reset_flags f) { + reset_port_count(); + join_helper::reset_inputs(my_inputs, f ); + } + + // all methods on input ports should be called under mutual exclusion from join_node_base. + + bool tuple_build_may_succeed() { + return !ports_with_no_items; + } + + bool try_to_make_tuple(output_type &out) { + if(ports_with_no_items) return false; + return join_helper::get_items(my_inputs, out); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool try_to_make_tuple(output_type &out, message_metainfo& metainfo) { + if(ports_with_no_items) return false; + return join_helper::get_items(my_inputs, out, metainfo); + } +#endif + + void tuple_accepted() { + reset_port_count(); + join_helper::reset_ports(my_inputs); + } + void tuple_rejected() { + // nothing to do. + } + + input_type my_inputs; + base_node_type *my_node; + std::atomic ports_with_no_items; + }; // join_node_FE + + // key_matching join front-end. + template + class join_node_FE, InputTuple, OutputTuple> : public matching_forwarding_base, + // buffer of key value counts + public hash_buffer< // typedefed below to key_to_count_buffer_type + typename std::decay::type&, // force ref type on K + count_element::type>, + type_to_key_function_body< + count_element::type>, + typename std::decay::type& >, + KHash >, + // buffer of output items + public item_buffer { + public: + static const int N = std::tuple_size::value; + typedef OutputTuple output_type; + typedef InputTuple input_type; + typedef K key_type; + typedef typename std::decay::type unref_key_type; + typedef KHash key_hash_compare; + // must use K without ref. + typedef count_element count_element_type; + // method that lets us refer to the key of this type. + typedef key_to_count_functor key_to_count_func; + typedef type_to_key_function_body< count_element_type, unref_key_type&> TtoK_function_body_type; + typedef type_to_key_function_body_leaf TtoK_function_body_leaf_type; + // this is the type of the special table that keeps track of the number of discrete + // elements corresponding to each key that we've seen. + typedef hash_buffer< unref_key_type&, count_element_type, TtoK_function_body_type, key_hash_compare > + key_to_count_buffer_type; + typedef item_buffer output_buffer_type; + typedef join_node_base, InputTuple, OutputTuple> base_node_type; // for forwarding + typedef matching_forwarding_base forwarding_base_type; + +// ----------- Aggregator ------------ + // the aggregator is only needed to serialize the access to the hash table. + // and the output_buffer_type base class + private: + enum op_type { res_count, inc_count, may_succeed, try_make }; + typedef join_node_FE, InputTuple, OutputTuple> class_type; + + class key_matching_FE_operation : public d1::aggregated_operation { + public: + char type; + unref_key_type my_val; + output_type* my_output; + graph_task* bypass_t; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo* metainfo = nullptr; +#endif + // constructor for value parameter + key_matching_FE_operation(const unref_key_type& e , op_type t) : type(char(t)), my_val(e), + my_output(nullptr), bypass_t(nullptr) {} + key_matching_FE_operation(output_type *p, op_type t) : type(char(t)), my_output(p), bypass_t(nullptr) {} +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + key_matching_FE_operation(output_type *p, op_type t, message_metainfo& info) + : type(char(t)), my_output(p), bypass_t(nullptr), metainfo(&info) {} +#endif + // constructor with no parameter + key_matching_FE_operation(op_type t) : type(char(t)), my_output(nullptr), bypass_t(nullptr) {} + }; + + typedef d1::aggregating_functor handler_type; + friend class d1::aggregating_functor; + d1::aggregator my_aggregator; + + // called from aggregator, so serialized + // returns a task pointer if the a task would have been enqueued but we asked that + // it be returned. Otherwise returns nullptr. + graph_task* fill_output_buffer(unref_key_type &t) { + output_type l_out; + graph_task* rtask = nullptr; + bool do_fwd = this->buffer_empty() && is_graph_active(this->graph_ref); + this->current_key = t; + this->delete_with_key(this->current_key); // remove the key +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo metainfo; +#endif + if(join_helper::get_items(my_inputs, l_out __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo))) { // <== call back + this->push_back(l_out __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + if(do_fwd) { // we enqueue if receiving an item from predecessor, not if successor asks for item + d1::small_object_allocator allocator{}; + typedef forward_task_bypass task_type; + rtask = allocator.new_object(this->graph_ref, allocator, *my_node); + do_fwd = false; + } + // retire the input values + join_helper::reset_ports(my_inputs); // <== call back + } + else { + __TBB_ASSERT(false, "should have had something to push"); + } + return rtask; + } + + void handle_operations(key_matching_FE_operation* op_list) { + key_matching_FE_operation *current; + while(op_list) { + current = op_list; + op_list = op_list->next; + switch(current->type) { + case res_count: // called from BE + { + this->destroy_front(); + current->status.store( SUCCEEDED, std::memory_order_release); + } + break; + case inc_count: { // called from input ports + count_element_type *p = nullptr; + unref_key_type &t = current->my_val; + if(!(this->find_ref_with_key(t,p))) { + count_element_type ev; + ev.my_key = t; + ev.my_value = 0; + this->insert_with_key(ev); + bool found = this->find_ref_with_key(t, p); + __TBB_ASSERT_EX(found, "should find key after inserting it"); + } + if(++(p->my_value) == size_t(N)) { + current->bypass_t = fill_output_buffer(t); + } + } + current->status.store( SUCCEEDED, std::memory_order_release); + break; + case may_succeed: // called from BE + current->status.store( this->buffer_empty() ? FAILED : SUCCEEDED, std::memory_order_release); + break; + case try_make: // called from BE + if(this->buffer_empty()) { + current->status.store( FAILED, std::memory_order_release); + } + else { + *(current->my_output) = this->front(); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (current->metainfo) { + *(current->metainfo) = this->front_metainfo(); + } +#endif + current->status.store( SUCCEEDED, std::memory_order_release); + } + break; + } + } + } +// ------------ End Aggregator --------------- + + public: + template + join_node_FE(graph &g, FunctionTuple &TtoK_funcs) : forwarding_base_type(g), my_node(nullptr) { + join_helper::set_join_node_pointer(my_inputs, this); + join_helper::set_key_functors(my_inputs, TtoK_funcs); + my_aggregator.initialize_handler(handler_type(this)); + TtoK_function_body_type *cfb = new TtoK_function_body_leaf_type(key_to_count_func()); + this->set_key_func(cfb); + } + + join_node_FE(const join_node_FE& other) : forwarding_base_type((other.forwarding_base_type::graph_ref)), key_to_count_buffer_type(), + output_buffer_type() { + my_node = nullptr; + join_helper::set_join_node_pointer(my_inputs, this); + join_helper::copy_key_functors(my_inputs, const_cast(other.my_inputs)); + my_aggregator.initialize_handler(handler_type(this)); + TtoK_function_body_type *cfb = new TtoK_function_body_leaf_type(key_to_count_func()); + this->set_key_func(cfb); + } + + // needed for forwarding + void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; } + + void reset_port_count() { // called from BE + key_matching_FE_operation op_data(res_count); + my_aggregator.execute(&op_data); + return; + } + + // if all input_ports have items, spawn forward to try and consume tuples + // return a task if we are asked and did create one. + graph_task *increment_key_count(unref_key_type const & t) override { // called from input_ports + key_matching_FE_operation op_data(t, inc_count); + my_aggregator.execute(&op_data); + return op_data.bypass_t; + } + + input_type &input_ports() { return my_inputs; } + + protected: + + void reset( reset_flags f ) { + // called outside of parallel contexts + join_helper::reset_inputs(my_inputs, f); + + key_to_count_buffer_type::reset(); + output_buffer_type::reset(); + } + + // all methods on input ports should be called under mutual exclusion from join_node_base. + + bool tuple_build_may_succeed() { // called from back-end + key_matching_FE_operation op_data(may_succeed); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + + // cannot lock while calling back to input_ports. current_key will only be set + // and reset under the aggregator, so it will remain consistent. + bool try_to_make_tuple(output_type &out) { + key_matching_FE_operation op_data(&out,try_make); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool try_to_make_tuple(output_type &out, message_metainfo& metainfo) { + key_matching_FE_operation op_data(&out, try_make, metainfo); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } +#endif + + void tuple_accepted() { + reset_port_count(); // reset current_key after ports reset. + } + + void tuple_rejected() { + // nothing to do. + } + + input_type my_inputs; // input ports + base_node_type *my_node; + }; // join_node_FE, InputTuple, OutputTuple> + + //! join_node_base + template + class join_node_base : public graph_node, public join_node_FE, + public sender { + protected: + using graph_node::my_graph; + public: + typedef OutputTuple output_type; + + typedef typename sender::successor_type successor_type; + typedef join_node_FE input_ports_type; + using input_ports_type::tuple_build_may_succeed; + using input_ports_type::try_to_make_tuple; + using input_ports_type::tuple_accepted; + using input_ports_type::tuple_rejected; + + private: + // ----------- Aggregator ------------ + enum op_type { reg_succ, rem_succ, try__get, do_fwrd, do_fwrd_bypass + }; + typedef join_node_base class_type; + + class join_node_base_operation : public d1::aggregated_operation { + public: + char type; + union { + output_type *my_arg; + successor_type *my_succ; + }; + graph_task* bypass_t; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo* metainfo; +#endif + join_node_base_operation(const output_type& e, op_type t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo& info)) + : type(char(t)), my_arg(const_cast(&e)), bypass_t(nullptr) + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo(&info)) {} +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + join_node_base_operation(const output_type& e, op_type t) + : type(char(t)), my_arg(const_cast(&e)), bypass_t(nullptr), metainfo(nullptr) {} +#endif + join_node_base_operation(const successor_type &s, op_type t) : type(char(t)), + my_succ(const_cast(&s)), bypass_t(nullptr) {} + join_node_base_operation(op_type t) : type(char(t)), bypass_t(nullptr) {} + }; + + typedef d1::aggregating_functor handler_type; + friend class d1::aggregating_functor; + bool forwarder_busy; + d1::aggregator my_aggregator; + + void handle_operations(join_node_base_operation* op_list) { + join_node_base_operation *current; + while(op_list) { + current = op_list; + op_list = op_list->next; + switch(current->type) { + case reg_succ: { + my_successors.register_successor(*(current->my_succ)); + if(tuple_build_may_succeed() && !forwarder_busy && is_graph_active(my_graph)) { + d1::small_object_allocator allocator{}; + typedef forward_task_bypass< join_node_base > task_type; + graph_task* t = allocator.new_object(my_graph, allocator, *this); + spawn_in_graph_arena(my_graph, *t); + forwarder_busy = true; + } + current->status.store( SUCCEEDED, std::memory_order_release); + } + break; + case rem_succ: + my_successors.remove_successor(*(current->my_succ)); + current->status.store( SUCCEEDED, std::memory_order_release); + break; + case try__get: + if(tuple_build_may_succeed()) { + bool make_tuple_result = false; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (current->metainfo) { + make_tuple_result = try_to_make_tuple(*(current->my_arg), *(current->metainfo)); + } else +#endif + { + make_tuple_result = try_to_make_tuple(*(current->my_arg)); + } + if(make_tuple_result) { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (current->metainfo) { + // Since elements would be removed from queues while calling to tuple_accepted + // together with corresponding message_metainfo objects + // we need to prolong the wait until the successor would create a task for removed elements + for (auto waiter : current->metainfo->waiters()) { + waiter->reserve(1); + } + } +#endif + tuple_accepted(); + current->status.store( SUCCEEDED, std::memory_order_release); + } + else current->status.store( FAILED, std::memory_order_release); + } + else current->status.store( FAILED, std::memory_order_release); + break; + case do_fwrd_bypass: { + bool build_succeeded; + graph_task *last_task = nullptr; + output_type out; + // forwarding must be exclusive, because try_to_make_tuple and tuple_accepted + // are separate locked methods in the FE. We could conceivably fetch the front + // of the FE queue, then be swapped out, have someone else consume the FE's + // object, then come back, forward, and then try to remove it from the queue + // again. Without reservation of the FE, the methods accessing it must be locked. + // We could remember the keys of the objects we forwarded, and then remove + // them from the input ports after forwarding is complete? + if(tuple_build_may_succeed()) { // checks output queue of FE + do { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo metainfo; +#endif + // fetch front_end of queue + build_succeeded = try_to_make_tuple(out __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + if(build_succeeded) { + graph_task *new_task = + my_successors.try_put_task(out __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + last_task = combine_tasks(my_graph, last_task, new_task); + if(new_task) { + tuple_accepted(); + } + else { + tuple_rejected(); + build_succeeded = false; + } + } + } while(build_succeeded); + } + current->bypass_t = last_task; + current->status.store( SUCCEEDED, std::memory_order_release); + forwarder_busy = false; + } + break; + } + } + } + // ---------- end aggregator ----------- + public: + join_node_base(graph &g) + : graph_node(g), input_ports_type(g), forwarder_busy(false), my_successors(this) + { + input_ports_type::set_my_node(this); + my_aggregator.initialize_handler(handler_type(this)); + } + + join_node_base(const join_node_base& other) : + graph_node(other.graph_node::my_graph), input_ports_type(other), + sender(), forwarder_busy(false), my_successors(this) + { + input_ports_type::set_my_node(this); + my_aggregator.initialize_handler(handler_type(this)); + } + + template + join_node_base(graph &g, FunctionTuple f) + : graph_node(g), input_ports_type(g, f), forwarder_busy(false), my_successors(this) + { + input_ports_type::set_my_node(this); + my_aggregator.initialize_handler(handler_type(this)); + } + + bool register_successor(successor_type &r) override { + join_node_base_operation op_data(r, reg_succ); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + + bool remove_successor( successor_type &r) override { + join_node_base_operation op_data(r, rem_succ); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + + bool try_get( output_type &v) override { + join_node_base_operation op_data(v, try__get); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool try_get( output_type &v, message_metainfo& metainfo) override { + join_node_base_operation op_data(v, try__get, metainfo); + my_aggregator.execute(&op_data); + return op_data.status == SUCCEEDED; + } +#endif + + protected: + void reset_node(reset_flags f) override { + input_ports_type::reset(f); + if(f & rf_clear_edges) my_successors.clear(); + } + + private: + broadcast_cache my_successors; + + friend class forward_task_bypass< join_node_base >; + graph_task *forward_task() { + join_node_base_operation op_data(do_fwrd_bypass); + my_aggregator.execute(&op_data); + return op_data.bypass_t; + } + + }; // join_node_base + + // join base class type generator + template class PortType, typename OutputTuple, typename JoinPolicy> + struct join_base { + using type = join_node_base::type, + OutputTuple>; + }; + + template + struct join_base > { + using key_type = K; + using key_hash_compare = KHash; + using key_traits_type = key_matching; + + using type = join_node_base::type, + OutputTuple>; + }; + +#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING + template + struct key_from_message_body { + K operator()(const T& t) const { + return key_from_message(t); + } + }; + // Adds const to reference type + template + struct key_from_message_body { + const K& operator()(const T& t) const { + return key_from_message(t); + } + }; +#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ + + //! unfolded_join_node : passes input_ports_type to join_node_base. We build the input port type + // using tuple_element. The class PortType is the port type (reserving_port, queueing_port, key_matching_port) + // and should match the typename. + template class PortType, typename OutputTuple, typename JoinPolicy> + class unfolded_join_node : public join_base::type { + public: + using input_ports_type = typename wrap_tuple_elements::type; + using output_type = OutputTuple; + private: + using base_type = join_node_base; + public: + unfolded_join_node(graph& g) : base_type(g) {} + unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} + }; + + // key_matching unfolded_join_node. This must be a separate specialization because the constructors + // differ. + template + class unfolded_join_node, key_matching> + : public join_base, key_matching>::type + { + public: + using input_ports_type = typename wrap_key_tuple_elements, + std::tuple>::type; + using output_type = std::tuple; + private: + using base_type = join_node_base, input_ports_type, output_type>; + using func_initializer_type = std::tuple*...>; + public: +#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING + unfolded_join_node(graph &g) : base_type(g, func_initializer_type( + new type_to_key_function_body_leaf> + (key_from_message_body())...)) + {} +#endif + template + unfolded_join_node(graph &g, Bodies... bodies) : base_type(g, func_initializer_type( + new type_to_key_function_body_leaf(bodies)...)) + {} + unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} + }; + + //! templated function to refer to input ports of the join node + template + typename std::tuple_element::type &input_port(JNT &jn) { + return std::get(jn.input_ports()); + } + +#endif // __TBB__flow_graph_join_impl_H diff --git a/src/3rd party/oneapi/tbb/detail/_flow_graph_node_impl.h b/src/3rd party/oneapi/tbb/detail/_flow_graph_node_impl.h new file mode 100644 index 0000000000..50e4b38e3b --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_flow_graph_node_impl.h @@ -0,0 +1,891 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__flow_graph_node_impl_H +#define __TBB__flow_graph_node_impl_H + +#ifndef __TBB_flow_graph_H +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +#include "_flow_graph_item_buffer_impl.h" + +template< typename T, typename A > +class function_input_queue : public item_buffer { +public: + bool empty() const { + return this->buffer_empty(); + } + + const T& front() const { + return this->item_buffer::front(); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + const message_metainfo& front_metainfo() const { + return this->item_buffer::front_metainfo(); + } +#endif + + void pop() { + this->destroy_front(); + } + + bool push( T& t ) { + return this->push_back( t ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool push( T& t, const message_metainfo& metainfo ) { + return this->push_back(t, metainfo); + } +#endif +}; + +//! Input and scheduling for a function node that takes a type Input as input +// The only up-ref is apply_body_impl, which should implement the function +// call and any handling of the result. +template< typename Input, typename Policy, typename A, typename ImplType > +class function_input_base : public receiver, no_assign { + enum op_type {reg_pred, rem_pred, try_fwd, tryput_bypass, app_body_bypass, occupy_concurrency + }; + typedef function_input_base class_type; + +public: + + //! The input type of this receiver + typedef Input input_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef predecessor_cache predecessor_cache_type; + typedef function_input_queue input_queue_type; + typedef typename allocator_traits::template rebind_alloc allocator_type; + static_assert(!has_policy::value || !has_policy::value, ""); + + //! Constructor for function_input_base + function_input_base( graph &g, size_t max_concurrency, node_priority_t a_priority, bool is_no_throw ) + : my_graph_ref(g), my_max_concurrency(max_concurrency) + , my_concurrency(0), my_priority(a_priority), my_is_no_throw(is_no_throw) + , my_queue(!has_policy::value ? new input_queue_type() : nullptr) + , my_predecessors(this) + , forwarder_busy(false) + { + my_aggregator.initialize_handler(handler_type(this)); + } + + //! Copy constructor + function_input_base( const function_input_base& src ) + : function_input_base(src.my_graph_ref, src.my_max_concurrency, src.my_priority, src.my_is_no_throw) {} + + //! Destructor + // The queue is allocated by the constructor for {multi}function_node. + // TODO: pass the graph_buffer_policy to the base so it can allocate the queue instead. + // This would be an interface-breaking change. + virtual ~function_input_base() { + delete my_queue; + my_queue = nullptr; + } + + graph_task* try_put_task( const input_type& t) override { + return try_put_task_base(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task( const input_type& t, const message_metainfo& metainfo ) override { + return try_put_task_base(t, metainfo); + } +#endif // __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + + //! Adds src to the list of cached predecessors. + bool register_predecessor( predecessor_type &src ) override { + operation_type op_data(reg_pred); + op_data.r = &src; + my_aggregator.execute(&op_data); + return true; + } + + //! Removes src from the list of cached predecessors. + bool remove_predecessor( predecessor_type &src ) override { + operation_type op_data(rem_pred); + op_data.r = &src; + my_aggregator.execute(&op_data); + return true; + } + +protected: + + void reset_function_input_base( reset_flags f) { + my_concurrency = 0; + if(my_queue) { + my_queue->reset(); + } + reset_receiver(f); + forwarder_busy = false; + } + + graph& my_graph_ref; + const size_t my_max_concurrency; + size_t my_concurrency; + node_priority_t my_priority; + const bool my_is_no_throw; + input_queue_type *my_queue; + predecessor_cache my_predecessors; + + void reset_receiver( reset_flags f) { + if( f & rf_clear_edges) my_predecessors.clear(); + else + my_predecessors.reset(); + __TBB_ASSERT(!(f & rf_clear_edges) || my_predecessors.empty(), "function_input_base reset failed"); + } + + graph& graph_reference() const override { + return my_graph_ref; + } + + graph_task* try_get_postponed_task(const input_type& i) { + operation_type op_data(i, app_body_bypass); // tries to pop an item or get_item + my_aggregator.execute(&op_data); + return op_data.bypass_t; + } + +private: + + friend class apply_body_task_bypass< class_type, input_type >; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + friend class apply_body_task_bypass< class_type, input_type, trackable_messages_graph_task >; +#endif + friend class forward_task_bypass< class_type >; + + class operation_type : public d1::aggregated_operation< operation_type > { + public: + char type; + union { + input_type *elem; + predecessor_type *r; + }; + graph_task* bypass_t; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo* metainfo; +#endif + operation_type(const input_type& e, op_type t) : + type(char(t)), elem(const_cast(&e)), bypass_t(nullptr) +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + , metainfo(nullptr) +#endif + {} +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + operation_type(const input_type& e, op_type t, const message_metainfo& info) : + type(char(t)), elem(const_cast(&e)), bypass_t(nullptr), + metainfo(const_cast(&info)) {} +#endif + operation_type(op_type t) : type(char(t)), r(nullptr), bypass_t(nullptr) {} + }; + + bool forwarder_busy; + typedef d1::aggregating_functor handler_type; + friend class d1::aggregating_functor; + d1::aggregator< handler_type, operation_type > my_aggregator; + + graph_task* perform_queued_requests() { + graph_task* new_task = nullptr; + if(my_queue) { + if(!my_queue->empty()) { + ++my_concurrency; + // TODO: consider removing metainfo from the queue using move semantics to avoid + // ref counter increase + new_task = create_body_task(my_queue->front() + __TBB_FLOW_GRAPH_METAINFO_ARG(my_queue->front_metainfo())); + + my_queue->pop(); + } + } + else { + input_type i; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo metainfo; +#endif + if(my_predecessors.get_item(i __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo))) { + ++my_concurrency; + new_task = create_body_task(i __TBB_FLOW_GRAPH_METAINFO_ARG(std::move(metainfo))); + } + } + return new_task; + } + void handle_operations(operation_type *op_list) { + operation_type* tmp; + while (op_list) { + tmp = op_list; + op_list = op_list->next; + switch (tmp->type) { + case reg_pred: + my_predecessors.add(*(tmp->r)); + tmp->status.store(SUCCEEDED, std::memory_order_release); + if (!forwarder_busy) { + forwarder_busy = true; + spawn_forward_task(); + } + break; + case rem_pred: + my_predecessors.remove(*(tmp->r)); + tmp->status.store(SUCCEEDED, std::memory_order_release); + break; + case app_body_bypass: { + tmp->bypass_t = nullptr; + __TBB_ASSERT(my_max_concurrency != 0, nullptr); + --my_concurrency; + if(my_concurrencybypass_t = perform_queued_requests(); + tmp->status.store(SUCCEEDED, std::memory_order_release); + } + break; + case tryput_bypass: internal_try_put_task(tmp); break; + case try_fwd: internal_forward(tmp); break; + case occupy_concurrency: + if (my_concurrency < my_max_concurrency) { + ++my_concurrency; + tmp->status.store(SUCCEEDED, std::memory_order_release); + } else { + tmp->status.store(FAILED, std::memory_order_release); + } + break; + } + } + } + + //! Put to the node, but return the task instead of enqueueing it + void internal_try_put_task(operation_type *op) { + __TBB_ASSERT(my_max_concurrency != 0, nullptr); + if (my_concurrency < my_max_concurrency) { + ++my_concurrency; + graph_task* new_task = create_body_task(*(op->elem) + __TBB_FLOW_GRAPH_METAINFO_ARG(*(op->metainfo))); + op->bypass_t = new_task; + op->status.store(SUCCEEDED, std::memory_order_release); + } else if ( my_queue && my_queue->push(*(op->elem) + __TBB_FLOW_GRAPH_METAINFO_ARG(*(op->metainfo))) ) + { + op->bypass_t = SUCCESSFULLY_ENQUEUED; + op->status.store(SUCCEEDED, std::memory_order_release); + } else { + op->bypass_t = nullptr; + op->status.store(FAILED, std::memory_order_release); + } + } + + //! Creates tasks for postponed messages if available and if concurrency allows + void internal_forward(operation_type *op) { + op->bypass_t = nullptr; + if (my_concurrency < my_max_concurrency) + op->bypass_t = perform_queued_requests(); + if(op->bypass_t) + op->status.store(SUCCEEDED, std::memory_order_release); + else { + forwarder_busy = false; + op->status.store(FAILED, std::memory_order_release); + } + } + + graph_task* internal_try_put_bypass( const input_type& t + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + { + operation_type op_data(t, tryput_bypass __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + my_aggregator.execute(&op_data); + if( op_data.status == SUCCEEDED ) { + return op_data.bypass_t; + } + return nullptr; + } + + graph_task* try_put_task_base(const input_type& t + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + { + if ( my_is_no_throw ) + return try_put_task_impl(t, has_policy() + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + else + return try_put_task_impl(t, std::false_type() + __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + } + + graph_task* try_put_task_impl( const input_type& t, /*lightweight=*/std::true_type + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + { + if( my_max_concurrency == 0 ) { + return apply_body_bypass(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + } else { + operation_type check_op(t, occupy_concurrency); + my_aggregator.execute(&check_op); + if( check_op.status == SUCCEEDED ) { + return apply_body_bypass(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + } + return internal_try_put_bypass(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + } + } + + graph_task* try_put_task_impl( const input_type& t, /*lightweight=*/std::false_type + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + { + if( my_max_concurrency == 0 ) { + return create_body_task(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + } else { + return internal_try_put_bypass(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + } + } + + //! Applies the body to the provided input + // then decides if more work is available + graph_task* apply_body_bypass( const input_type &i + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + + { + return static_cast(this)->apply_body_impl_bypass(i __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + } + + //! allocates a task to apply a body +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + graph_task* create_body_task( const input_type &input, Metainfo&& metainfo ) +#else + graph_task* create_body_task( const input_type &input ) +#endif + { + if (!is_graph_active(my_graph_ref)) { + return nullptr; + } + // TODO revamp: extract helper for common graph task allocation part + d1::small_object_allocator allocator{}; + graph_task* t = nullptr; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (!metainfo.empty()) { + using task_type = apply_body_task_bypass; + t = allocator.new_object(my_graph_ref, allocator, *this, input, my_priority, std::forward(metainfo)); + } else +#endif + { + using task_type = apply_body_task_bypass; + t = allocator.new_object(my_graph_ref, allocator, *this, input, my_priority); + } + return t; + } + + //! This is executed by an enqueued task, the "forwarder" + graph_task* forward_task() { + operation_type op_data(try_fwd); + graph_task* rval = nullptr; + do { + op_data.status = WAIT; + my_aggregator.execute(&op_data); + if(op_data.status == SUCCEEDED) { + graph_task* ttask = op_data.bypass_t; + __TBB_ASSERT( ttask && ttask != SUCCESSFULLY_ENQUEUED, nullptr); + rval = combine_tasks(my_graph_ref, rval, ttask); + } + } while (op_data.status == SUCCEEDED); + return rval; + } + + inline graph_task* create_forward_task() { + if (!is_graph_active(my_graph_ref)) { + return nullptr; + } + d1::small_object_allocator allocator{}; + typedef forward_task_bypass task_type; + graph_task* t = allocator.new_object( graph_reference(), allocator, *this, my_priority ); + return t; + } + + //! Spawns a task that calls forward() + inline void spawn_forward_task() { + graph_task* tp = create_forward_task(); + if(tp) { + spawn_in_graph_arena(graph_reference(), *tp); + } + } + + node_priority_t priority() const override { return my_priority; } +}; // function_input_base + +//! Implements methods for a function node that takes a type Input as input and sends +// a type Output to its successors. +template< typename Input, typename Output, typename Policy, typename A> +class function_input : public function_input_base > { +public: + typedef Input input_type; + typedef Output output_type; + typedef function_body function_body_type; + typedef function_input my_class; + typedef function_input_base base_type; + typedef function_input_queue input_queue_type; + + // constructor + template + function_input( + graph &g, size_t max_concurrency, Body& body, node_priority_t a_priority ) + : base_type(g, max_concurrency, a_priority, noexcept(tbb::detail::invoke(body, input_type()))) + , my_body( new function_body_leaf< input_type, output_type, Body>(body) ) + , my_init_body( new function_body_leaf< input_type, output_type, Body>(body) ) { + } + + //! Copy constructor + function_input( const function_input& src ) : + base_type(src), + my_body( src.my_init_body->clone() ), + my_init_body(src.my_init_body->clone() ) { + } +#if __INTEL_COMPILER <= 2021 + // Suppress superfluous diagnostic about virtual keyword absence in a destructor of an inherited + // class while the parent class has the virtual keyword for the destrocutor. + virtual +#endif + ~function_input() { + delete my_body; + delete my_init_body; + } + + template< typename Body > + Body copy_function_object() { + function_body_type &body_ref = *this->my_body; + return dynamic_cast< function_body_leaf & >(body_ref).get_body(); + } + + output_type apply_body_impl( const input_type& i) { + // There is an extra copied needed to capture the + // body execution without the try_put + fgt_begin_body( my_body ); + output_type v = tbb::detail::invoke(*my_body, i); + fgt_end_body( my_body ); + return v; + } + + //TODO: consider moving into the base class + graph_task* apply_body_impl_bypass( const input_type &i + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) + { + output_type v = apply_body_impl(i); + graph_task* postponed_task = nullptr; + if( base_type::my_max_concurrency != 0 ) { + postponed_task = base_type::try_get_postponed_task(i); + __TBB_ASSERT( !postponed_task || postponed_task != SUCCESSFULLY_ENQUEUED, nullptr); + } + if( postponed_task ) { + // make the task available for other workers since we do not know successors' + // execution policy + spawn_in_graph_arena(base_type::graph_reference(), *postponed_task); + } + graph_task* successor_task = successors().try_put_task(v __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); +#if _MSC_VER && !__INTEL_COMPILER +#pragma warning (push) +#pragma warning (disable: 4127) /* suppress conditional expression is constant */ +#endif + if(has_policy::value) { +#if _MSC_VER && !__INTEL_COMPILER +#pragma warning (pop) +#endif + if(!successor_task) { + // Return confirmative status since current + // node's body has been executed anyway + successor_task = SUCCESSFULLY_ENQUEUED; + } + } + return successor_task; + } + +protected: + + void reset_function_input(reset_flags f) { + base_type::reset_function_input_base(f); + if(f & rf_reset_bodies) { + function_body_type *tmp = my_init_body->clone(); + delete my_body; + my_body = tmp; + } + } + + function_body_type *my_body; + function_body_type *my_init_body; + virtual broadcast_cache &successors() = 0; + +}; // function_input + + +// helper templates to clear the successor edges of the output ports of an multifunction_node +template struct clear_element { + template static void clear_this(P &p) { + (void)std::get(p).successors().clear(); + clear_element::clear_this(p); + } +#if TBB_USE_ASSERT + template static bool this_empty(P &p) { + if(std::get(p).successors().empty()) + return clear_element::this_empty(p); + return false; + } +#endif +}; + +template <> struct clear_element<0> { + template static void clear_this(P&) {} +#if TBB_USE_ASSERT + template static bool this_empty(P&) { return true; } +#endif +}; + +template +struct init_output_ports { + template + static OutputTuple call(graph& g, const std::tuple&) { + return OutputTuple(Args(g)...); + } +}; // struct init_output_ports + +//! Implements methods for a function node that takes a type Input as input +// and has a tuple of output ports specified. +template< typename Input, typename OutputPortSet, typename Policy, typename A> +class multifunction_input : public function_input_base > { +public: + static const int N = std::tuple_size::value; + typedef Input input_type; + typedef OutputPortSet output_ports_type; + typedef multifunction_body multifunction_body_type; + typedef multifunction_input my_class; + typedef function_input_base base_type; + typedef function_input_queue input_queue_type; + + // constructor + template + multifunction_input(graph &g, size_t max_concurrency,Body& body, node_priority_t a_priority ) + : base_type(g, max_concurrency, a_priority, noexcept(tbb::detail::invoke(body, input_type(), my_output_ports))) + , my_body( new multifunction_body_leaf(body) ) + , my_init_body( new multifunction_body_leaf(body) ) + , my_output_ports(init_output_ports::call(g, my_output_ports)){ + } + + //! Copy constructor + multifunction_input( const multifunction_input& src ) : + base_type(src), + my_body( src.my_init_body->clone() ), + my_init_body(src.my_init_body->clone() ), + my_output_ports( init_output_ports::call(src.my_graph_ref, my_output_ports) ) { + } + + ~multifunction_input() { + delete my_body; + delete my_init_body; + } + + template< typename Body > + Body copy_function_object() { + multifunction_body_type &body_ref = *this->my_body; + return *static_cast(dynamic_cast< multifunction_body_leaf & >(body_ref).get_body_ptr()); + } + + // for multifunction nodes we do not have a single successor as such. So we just tell + // the task we were successful. + //TODO: consider moving common parts with implementation in function_input into separate function + graph_task* apply_body_impl_bypass( const input_type &i + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo&) ) + { + fgt_begin_body( my_body ); + (*my_body)(i, my_output_ports); + fgt_end_body( my_body ); + graph_task* ttask = nullptr; + if(base_type::my_max_concurrency != 0) { + ttask = base_type::try_get_postponed_task(i); + } + return ttask ? ttask : SUCCESSFULLY_ENQUEUED; + } + + output_ports_type &output_ports(){ return my_output_ports; } + +protected: + + void reset(reset_flags f) { + base_type::reset_function_input_base(f); + if(f & rf_clear_edges)clear_element::clear_this(my_output_ports); + if(f & rf_reset_bodies) { + multifunction_body_type* tmp = my_init_body->clone(); + delete my_body; + my_body = tmp; + } + __TBB_ASSERT(!(f & rf_clear_edges) || clear_element::this_empty(my_output_ports), "multifunction_node reset failed"); + } + + multifunction_body_type *my_body; + multifunction_body_type *my_init_body; + output_ports_type my_output_ports; + +}; // multifunction_input + +// template to refer to an output port of a multifunction_node +template +typename std::tuple_element::type &output_port(MOP &op) { + return std::get(op.output_ports()); +} + +inline void check_task_and_spawn(graph& g, graph_task* t) { + if (t && t != SUCCESSFULLY_ENQUEUED) { + spawn_in_graph_arena(g, *t); + } +} + +// helper structs for split_node +template +struct emit_element { + template + static graph_task* emit_this(graph& g, const T &t, P &p) { + // TODO: consider to collect all the tasks in task_list and spawn them all at once + graph_task* last_task = std::get(p).try_put_task(std::get(t)); + check_task_and_spawn(g, last_task); + return emit_element::emit_this(g,t,p); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + static graph_task* emit_this(graph& g, const TupleType& t, PortsType& p, + const message_metainfo& metainfo) + { + // TODO: consider to collect all the tasks in task_list and spawn them all at once + graph_task* last_task = std::get(p).try_put_task(std::get(t), metainfo); + check_task_and_spawn(g, last_task); + return emit_element::emit_this(g, t, p, metainfo); + } +#endif +}; + +template<> +struct emit_element<1> { + template + static graph_task* emit_this(graph& g, const T &t, P &p) { + graph_task* last_task = std::get<0>(p).try_put_task(std::get<0>(t)); + check_task_and_spawn(g, last_task); + return SUCCESSFULLY_ENQUEUED; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + template + static graph_task* emit_this(graph& g, const TupleType& t, PortsType& ports, + const message_metainfo& metainfo) + { + graph_task* last_task = std::get<0>(ports).try_put_task(std::get<0>(t), metainfo); + check_task_and_spawn(g, last_task); + return SUCCESSFULLY_ENQUEUED; + } +#endif +}; + +//! Implements methods for an executable node that takes continue_msg as input +template< typename Output, typename Policy> +class continue_input : public continue_receiver { +public: + + //! The input type of this receiver + typedef continue_msg input_type; + + //! The output type of this receiver + typedef Output output_type; + typedef function_body function_body_type; + typedef continue_input class_type; + + template< typename Body > + continue_input( graph &g, Body& body, node_priority_t a_priority ) + : continue_receiver(/*number_of_predecessors=*/0, a_priority) + , my_graph_ref(g) + , my_body( new function_body_leaf< input_type, output_type, Body>(body) ) + , my_init_body( new function_body_leaf< input_type, output_type, Body>(body) ) + { } + + template< typename Body > + continue_input( graph &g, int number_of_predecessors, + Body& body, node_priority_t a_priority ) + : continue_receiver( number_of_predecessors, a_priority ) + , my_graph_ref(g) + , my_body( new function_body_leaf< input_type, output_type, Body>(body) ) + , my_init_body( new function_body_leaf< input_type, output_type, Body>(body) ) + { } + + continue_input( const continue_input& src ) : continue_receiver(src), + my_graph_ref(src.my_graph_ref), + my_body( src.my_init_body->clone() ), + my_init_body( src.my_init_body->clone() ) {} + + ~continue_input() { + delete my_body; + delete my_init_body; + } + + template< typename Body > + Body copy_function_object() { + function_body_type &body_ref = *my_body; + return dynamic_cast< function_body_leaf & >(body_ref).get_body(); + } + + void reset_receiver( reset_flags f) override { + continue_receiver::reset_receiver(f); + if(f & rf_reset_bodies) { + function_body_type *tmp = my_init_body->clone(); + delete my_body; + my_body = tmp; + } + } + +protected: + + graph& my_graph_ref; + function_body_type *my_body; + function_body_type *my_init_body; + + virtual broadcast_cache &successors() = 0; + + friend class apply_body_task_bypass< class_type, continue_msg >; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + friend class apply_body_task_bypass< class_type, continue_msg, trackable_messages_graph_task >; +#endif + + //! Applies the body to the provided input + graph_task* apply_body_bypass( input_type __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo) ) { + // There is an extra copied needed to capture the + // body execution without the try_put + fgt_begin_body( my_body ); + output_type v = (*my_body)( continue_msg() ); + fgt_end_body( my_body ); + return successors().try_put_task( v __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* execute(const message_metainfo& metainfo) override { +#else + graph_task* execute() override { +#endif + if(!is_graph_active(my_graph_ref)) { + return nullptr; + } +#if _MSC_VER && !__INTEL_COMPILER +#pragma warning (push) +#pragma warning (disable: 4127) /* suppress conditional expression is constant */ +#endif + if(has_policy::value) { +#if _MSC_VER && !__INTEL_COMPILER +#pragma warning (pop) +#endif + return apply_body_bypass( continue_msg() __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo) ); + } + else { + d1::small_object_allocator allocator{}; + graph_task* t = nullptr; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (!metainfo.empty()) { + using task_type = apply_body_task_bypass; + t = allocator.new_object( graph_reference(), allocator, *this, continue_msg(), my_priority, metainfo ); + } else +#endif + { + using task_type = apply_body_task_bypass; + t = allocator.new_object( graph_reference(), allocator, *this, continue_msg(), my_priority ); + } + return t; + } + } + + graph& graph_reference() const override { + return my_graph_ref; + } +}; // continue_input + +//! Implements methods for both executable and function nodes that puts Output to its successors +template< typename Output > +class function_output : public sender { +public: + + template friend struct clear_element; + typedef Output output_type; + typedef typename sender::successor_type successor_type; + typedef broadcast_cache broadcast_cache_type; + + function_output(graph& g) : my_successors(this), my_graph_ref(g) {} + function_output(const function_output& other) = delete; + + //! Adds a new successor to this node + bool register_successor( successor_type &r ) override { + successors().register_successor( r ); + return true; + } + + //! Removes a successor from this node + bool remove_successor( successor_type &r ) override { + successors().remove_successor( r ); + return true; + } + + broadcast_cache_type &successors() { return my_successors; } + + graph& graph_reference() const { return my_graph_ref; } +protected: + broadcast_cache_type my_successors; + graph& my_graph_ref; +}; // function_output + +template< typename Output > +class multifunction_output : public function_output { +public: + typedef Output output_type; + typedef function_output base_type; + using base_type::my_successors; + + multifunction_output(graph& g) : base_type(g) {} + multifunction_output(const multifunction_output& other) : base_type(other.my_graph_ref) {} + + bool try_put(const output_type &i) { + graph_task *res = try_put_task(i); + if( !res ) return false; + if( res != SUCCESSFULLY_ENQUEUED ) { + // wrapping in task_arena::execute() is not needed since the method is called from + // inside task::execute() + spawn_in_graph_arena(graph_reference(), *res); + } + return true; + } + + using base_type::graph_reference; + +protected: + + graph_task* try_put_task(const output_type &i) { + return my_successors.try_put_task(i); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const output_type& i, const message_metainfo& metainfo) { + return my_successors.try_put_task(i, metainfo); + } +#endif + + template friend struct emit_element; + +}; // multifunction_output + +//composite_node +template +void add_nodes_impl(CompositeType*, bool) {} + +template< typename CompositeType, typename NodeType1, typename... NodeTypes > +void add_nodes_impl(CompositeType *c_node, bool visible, const NodeType1& n1, const NodeTypes&... n) { + void *addr = const_cast(&n1); + + fgt_alias_port(c_node, addr, visible); + add_nodes_impl(c_node, visible, n...); +} + +#endif // __TBB__flow_graph_node_impl_H diff --git a/src/3rd party/tbb/internal/_flow_graph_node_set_impl.h b/src/3rd party/oneapi/tbb/detail/_flow_graph_node_set_impl.h similarity index 84% rename from src/3rd party/tbb/internal/_flow_graph_node_set_impl.h rename to src/3rd party/oneapi/tbb/detail/_flow_graph_node_set_impl.h index 6ca4544b18..8440bd7008 100644 --- a/src/3rd party/tbb/internal/_flow_graph_node_set_impl.h +++ b/src/3rd party/oneapi/tbb/detail/_flow_graph_node_set_impl.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2019 Intel Corporation + Copyright (c) 2020-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,9 +21,7 @@ #error Do not #include this internal file directly; use public TBB headers instead. #endif -// Included in namespace tbb::flow::interfaceX (in flow_graph.h) - -namespace internal { +// Included in namespace tbb::detail::d2 (in flow_graph.h) #if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET // Visual Studio 2019 reports an error while calling predecessor_selector::get and successor_selector::get @@ -63,7 +61,7 @@ template struct node_set { typedef Order order_type; - tbb::flow::tuple nodes; + std::tuple nodes; node_set(Nodes&... ns) : nodes(ns...) {} template @@ -82,16 +80,16 @@ template using input_ports_type = typename T::input_ports_type; } // namespace alias_helpers template -using has_output_type = tbb::internal::supports; +using has_output_type = supports; template -using has_input_type = tbb::internal::supports; +using has_input_type = supports; template -using has_input_ports_type = tbb::internal::supports; +using has_input_ports_type = supports; template -using has_output_ports_type = tbb::internal::supports; +using has_output_ports_type = supports; template struct is_sender : std::is_base_of, T> {}; @@ -103,15 +101,15 @@ template struct is_async_node : std::false_type {}; template -struct is_async_node> : std::true_type {}; +struct is_async_node> : std::true_type {}; template node_set follows(FirstPredecessor& first_predecessor, Predecessors&... predecessors) { - __TBB_STATIC_ASSERT((tbb::internal::conjunction, + static_assert((conjunction, has_output_type...>::value), "Not all node's predecessors has output_type typedef"); - __TBB_STATIC_ASSERT((tbb::internal::conjunction, is_sender...>::value), + static_assert((conjunction, is_sender...>::value), "Not all node's predecessors are senders"); return node_set(first_predecessor, predecessors...); } @@ -119,9 +117,9 @@ follows(FirstPredecessor& first_predecessor, Predecessors&... predecessors) { template node_set follows(node_set& predecessors_set) { - __TBB_STATIC_ASSERT((tbb::internal::conjunction...>::value), + static_assert((conjunction...>::value), "Not all nodes in the set has output_type typedef"); - __TBB_STATIC_ASSERT((tbb::internal::conjunction...>::value), + static_assert((conjunction...>::value), "Not all nodes in the set are senders"); return node_set(predecessors_set); } @@ -129,10 +127,10 @@ follows(node_set& predecessors_set) { template node_set precedes(FirstSuccessor& first_successor, Successors&... successors) { - __TBB_STATIC_ASSERT((tbb::internal::conjunction, + static_assert((conjunction, has_input_type...>::value), "Not all node's successors has input_type typedef"); - __TBB_STATIC_ASSERT((tbb::internal::conjunction, is_receiver...>::value), + static_assert((conjunction, is_receiver...>::value), "Not all node's successors are receivers"); return node_set(first_successor, successors...); } @@ -140,9 +138,9 @@ precedes(FirstSuccessor& first_successor, Successors&... successors) { template node_set precedes(node_set& successors_set) { - __TBB_STATIC_ASSERT((tbb::internal::conjunction...>::value), + static_assert((conjunction...>::value), "Not all nodes in the set has input_type typedef"); - __TBB_STATIC_ASSERT((tbb::internal::conjunction...>::value), + static_assert((conjunction...>::value), "Not all nodes in the set are receivers"); return node_set(successors_set); } @@ -170,7 +168,7 @@ class successor_selector { #else static auto get(NodeType& node) -> decltype(get_impl(node, has_input_ports_type())) #endif - { + { return get_impl(node, has_input_ports_type()); } }; @@ -191,7 +189,7 @@ class predecessor_selector { #else static auto get_impl(NodeType& node, std::false_type) -> decltype(internal_get(node, has_output_ports_type())) #endif - { + { return internal_get(node, has_output_ports_type()); } @@ -205,7 +203,7 @@ class predecessor_selector { #else static auto get(NodeType& node) -> decltype(get_impl(node, is_async_node())) #endif - { + { return get_impl(node, is_async_node()); } }; @@ -242,13 +240,13 @@ struct make_edges_helper<0> { // TODO: consider adding an overload for making edges between node sets template void make_edges(const node_set& s, NodeType& node) { - const std::size_t SetSize = tbb::flow::tuple_size::value; + const std::size_t SetSize = std::tuple_size::value; make_edges_helper::connect_predecessors(s.nodes, node); } template void make_edges(NodeType& node, const node_set& s) { - const std::size_t SetSize = tbb::flow::tuple_size::value; + const std::size_t SetSize = std::tuple_size::value; make_edges_helper::connect_successors(node, s.nodes); } @@ -262,8 +260,6 @@ void make_edges_in_order(const node_set& ns, NodeTyp make_edges(node, ns); } -#endif // __TBB_CPP11_PRESENT - -} // namespace internal +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET #endif // __TBB_flow_graph_node_set_impl_H diff --git a/src/3rd party/oneapi/tbb/detail/_flow_graph_nodes_deduction.h b/src/3rd party/oneapi/tbb/detail/_flow_graph_nodes_deduction.h new file mode 100644 index 0000000000..ce707bec74 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_flow_graph_nodes_deduction.h @@ -0,0 +1,300 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_flow_graph_nodes_deduction_H +#define __TBB_flow_graph_nodes_deduction_H + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +namespace tbb { +namespace detail { +namespace d2 { + +template +struct body_types { + using input_type = std::decay_t; + using output_type = std::decay_t; +}; + +template +struct checked_body_types : body_types { +private: + using base_type = body_types; + + static_assert(std::is_same_v || + std::is_same_v, + "The node body can only accept input by value or by const lvalue reference"); +}; + +template +struct body_traits { + // !std::is_same_v is needed to create a dependent context for static_assert(false) + static_assert(!std::is_same_v, "Body signature does not match the named requirements"); +}; + +// Body is a pointer to function +template +struct body_traits : body_types {}; + +// Body is a pointer to noexcept function +template +struct body_traits : body_types {}; + +// Body is a pointer to non-static data member +template +struct body_traits : body_types {}; + +// Body is a pointer to non-static member function +template +struct body_traits : body_types {}; + +template +struct body_traits : body_types {}; + +// Body is a callable/lambda +// Helper to read input/output types of operator() with single argument +// Supports Body defining multiple operator() overloads, but should have single unary overload +template +struct unary_operator_types_extractor { + // Overloads for const, noexcept and & qualified operator() with all possible combinations + template + static auto check_args(Output (B::*)(Input)) -> checked_body_types; + + template + static auto check_args(Output (B::*)(Input) const) -> checked_body_types; + + template + static auto check_args(Output (B::*)(Input) noexcept) -> checked_body_types; + + template + static auto check_args(Output (B::*)(Input) const noexcept) -> checked_body_types; + + template + static auto check_args(Output (B::*)(Input) &) -> checked_body_types; + + template + static auto check_args(Output (B::*)(Input) const &) -> checked_body_types; + + template + static auto check_args(Output (B::*)(Input) & noexcept) -> checked_body_types; + + template + static auto check_args(Output (B::*)(Input) const & noexcept) -> checked_body_types; + + // Checking layer to force SFINAE in case of operator() absence + template + static auto check_call_operator_presence(int) -> decltype(check_args(&B::operator())); + + template + static void check_call_operator_presence(...); + + using operator_types = decltype(check_call_operator_presence(0)); +}; + +template +struct body_traits::operator_types::input_type>> + : unary_operator_types_extractor::operator_types +{}; + +template +using input_type_of = typename body_traits::input_type; + +template +using output_type_of = typename body_traits::output_type; + +// Deduction guides for Flow Graph nodes + +template +input_node(GraphOrSet&&, Body) +->input_node>; + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + +template +struct decide_on_set; + +template +struct decide_on_set> { + using type = typename Node::output_type; +}; + +template +struct decide_on_set> { + using type = typename Node::input_type; +}; + +template +using decide_on_set_t = typename decide_on_set>::type; + +template +broadcast_node(const NodeSet&) +->broadcast_node>; + +template +buffer_node(const NodeSet&) +->buffer_node>; + +template +queue_node(const NodeSet&) +->queue_node>; +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + +template +sequencer_node(GraphOrProxy&&, Sequencer) +->sequencer_node>; + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET +template +priority_queue_node(const NodeSet&, const Compare&) +->priority_queue_node, Compare>; + +template +priority_queue_node(const NodeSet&) +->priority_queue_node, std::less>>; +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + +template +using is_queueing_or_reserving_tag = std::disjunction, std::is_same>; + +template +using are_queueing_or_reserving_tags = std::conjunction, + is_queueing_or_reserving_tag...>; + +template +struct join_key { + using type = Key; +}; + +template +struct join_key { + using type = T&; +}; + +template +using join_key_t = typename join_key::type; + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + +template ::value, int> = 0> +join_node(const node_set&, Policy) +->join_node, Policy>; + +template ::value, int> = 0> +join_node(const node_set&, Policy) +->join_node; + +template +join_node(const node_set) +->join_node, + queueing>; + +template +join_node(const node_set) +->join_node; +#endif + +template ::value, int> = 0> +join_node(GraphOrProxy&&, Body, Bodies...) +->join_node, input_type_of...>, + key_matching>>>; + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET +template +indexer_node(const node_set&) +->indexer_node; + +template +limiter_node(const NodeSet&, size_t) +->limiter_node>; + +template +split_node(const node_set&) +->split_node; + +template +split_node(const node_set&) +->split_node>; +#endif + +template +function_node(GraphOrSet&&, + size_t, Body, + Policy, node_priority_t = no_priority) +->function_node, output_type_of, Policy>; + +template +function_node(GraphOrSet&&, size_t, + Body, node_priority_t = no_priority) +->function_node, output_type_of, queueing>; + +template +struct continue_output { + using type = Output; +}; + +template <> +struct continue_output { + using type = continue_msg; +}; + +template +using continue_output_t = typename continue_output::type; + +template +continue_node(GraphOrSet&&, Body, + Policy, node_priority_t = no_priority) +->continue_node>, + Policy>; + +template +continue_node(GraphOrSet&&, + int, Body, + Policy, node_priority_t = no_priority) +->continue_node>, + Policy>; + +template +continue_node(GraphOrSet&&, + Body, node_priority_t = no_priority) +->continue_node>, Policy>; + +template +continue_node(GraphOrSet&&, int, + Body, node_priority_t = no_priority) +->continue_node>, + Policy>; + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + +template +overwrite_node(const NodeSet&) +->overwrite_node>; + +template +write_once_node(const NodeSet&) +->write_once_node>; +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET +} // namespace d2 +} // namespace detail +} // namespace tbb + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +#endif // __TBB_flow_graph_nodes_deduction_H diff --git a/src/3rd party/oneapi/tbb/detail/_flow_graph_resource_limiting.h b/src/3rd party/oneapi/tbb/detail/_flow_graph_resource_limiting.h new file mode 100644 index 0000000000..d4d28519e3 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_flow_graph_resource_limiting.h @@ -0,0 +1,663 @@ +/* + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__flow_graph_resource_limiting_H +#define __TBB__flow_graph_resource_limiting_H + +#ifndef __TBB_flow_graph_H +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +#include +#include +#include +#include +#include +#include + +namespace tbb { +namespace detail { +namespace d2 { + +template +class resource_consumer_base; + +class request_id { + std::uint64_t m_unique_integer; +public: + request_id(const std::uint64_t& unique_integer) + : m_unique_integer(unique_integer) + {} + + struct hash : protected std::hash { + std::size_t operator()(request_id id) const { + return std::hash::operator()(id.m_unique_integer); + } + }; + + struct equal : protected std::equal_to { + bool operator()(request_id lhs, request_id rhs) const { + return std::equal_to::operator()(lhs.m_unique_integer, rhs.m_unique_integer); + } + }; +}; // class request_id + +template +class resource_handle_optional { + union { + ResourceHandle m_resource_handle; + }; + bool m_has_value; + + template + void construct(Args&&... args) { + ::new(&m_resource_handle) ResourceHandle(std::forward(args)...); + } +public: + struct in_place_t {}; + + resource_handle_optional() + : m_has_value(false) + {} + + template + resource_handle_optional(in_place_t, Args&&... args) + : m_has_value(true) + { + construct(std::forward(args)...); + } + + resource_handle_optional(const resource_handle_optional&) = delete; + resource_handle_optional& operator=(const resource_handle_optional&) = delete; + + resource_handle_optional(resource_handle_optional&& other) + : m_has_value(other.m_has_value) + { + if (m_has_value) { + construct(std::move(other.m_resource_handle)); + } + } + + resource_handle_optional& operator=(resource_handle_optional&& other) { + if (this != &other) { + if (m_has_value) m_resource_handle.~ResourceHandle(); + m_has_value = other.m_has_value; + if (other.m_has_value) { + construct(std::move(other.m_resource_handle)); + } + } + return *this; + } + + ~resource_handle_optional() { + if (m_has_value) { + m_resource_handle.~ResourceHandle(); + } + } + + bool has_value() const { return m_has_value; } + + ResourceHandle& value() { + __TBB_ASSERT(has_value(), nullptr); + return m_resource_handle; + } +}; // class resource_handle_optional + +template +class resource_provider_base { +public: + using consumer_type = resource_consumer_base; + using optional_type = resource_handle_optional; + + virtual void request(consumer_type&, request_id) = 0; + virtual optional_type acquire(consumer_type&, request_id) = 0; + virtual void report_pressure(consumer_type&, std::size_t) = 0; + virtual void release(consumer_type&, request_id, optional_type&&) = 0; + virtual ~resource_provider_base() = default; +}; + +template +class resource_consumer_base { +public: + using provider_type = resource_provider_base; + virtual void notify(provider_type&, request_id) = 0; + virtual ~resource_consumer_base() = default; +}; + +// TODO: use actual fair implementation with starvation avoidance +template +class resource_limiter : public resource_provider_base { +public: + using resource_handle_type = ResourceHandle; + using consumer_type = typename resource_provider_base::consumer_type; + using optional_type = typename resource_provider_base::optional_type; + + template + resource_limiter(Handle&& handle, Handles&&... handles) { + emplace_handles(std::forward(handle), std::forward(handles)...); + } + + void request(consumer_type& consumer, request_id id) override { + // TODO: consider using an aggregator instead of mutex + tbb::spin_mutex::scoped_lock lock(m_mutex); + + if (m_resource_handles.empty()) { + m_consumers.emplace_front(std::piecewise_construct, std::forward_as_tuple(id), std::forward_as_tuple(&consumer)); + } else { + // Resource available immediately + lock.release(); + consumer.notify(*this, id); + } + } + + optional_type acquire(consumer_type& consumer, request_id id) override { + tbb::spin_mutex::scoped_lock lock(m_mutex); + + if (m_resource_handles.empty()) { + m_consumers.emplace_front(std::piecewise_construct, std::forward_as_tuple(id), std::forward_as_tuple(&consumer)); + return optional_type{}; + } else { + ResourceHandle handle = std::move(m_resource_handles.front()); + m_resource_handles.pop_front(); + return {typename optional_type::in_place_t{}, std::move(handle)}; + } + } + + void release(consumer_type&, request_id, optional_type&& handle) override { + __TBB_ASSERT(handle.has_value(), nullptr); + tbb::spin_mutex::scoped_lock lock(m_mutex); + + m_resource_handles.emplace_front(std::move(handle.value())); + + auto consumers = std::move(m_consumers); + m_consumers.clear(); + + lock.release(); + for (auto consumer_dt : consumers) { + consumer_dt.second->notify(*this, consumer_dt.first); + } + } + + void report_pressure(consumer_type&, std::size_t) override {} + + using consumer_data = std::pair*>; + +private: + template + void emplace_handles(Handle&& handle, Handles&&... handles) { + m_resource_handles.emplace_front(std::forward(handle)); + emplace_handles(std::forward(handles)...); + } + + void emplace_handles() {} + + tbb::spin_mutex m_mutex; + std::forward_list m_resource_handles; + std::forward_list m_consumers; +}; // class resource_limiter + +template +class resource_limited_body { + graph& m_graph; +public: + virtual void operator()(const Input& input, OutputPorts& ports) = 0; + virtual void notify(request_id id) = 0; + virtual resource_limited_body* clone() = 0; + virtual void* get_body_ptr() = 0; + virtual ~resource_limited_body() = default; + + resource_limited_body(graph& g) : m_graph(g) {} + + graph& graph_reference() { return m_graph; } +}; + +template +class resource_consumer : public resource_consumer_base { +public: + using resource_handle_type = typename ResourceProvider::resource_handle_type; + using resource_limited_body_type = resource_limited_body; + using optional_type = typename ResourceProvider::optional_type; + + resource_consumer(ResourceProvider& provider, resource_limited_body_type* body_ptr) + : m_resource_provider(provider) + , m_body_ptr(body_ptr) + {} + + void request_from_provider(request_id id) { + m_resource_provider.request(*this, id); + } + + + optional_type acquire_from_provider(request_id id) { + return m_resource_provider.acquire(*this, id); + } + + void release_to_provider(request_id id, optional_type&& handle) { + m_resource_provider.release(*this, id, std::move(handle)); + } + + void report_pressure_to_provider(std::size_t pressure) { + m_resource_provider.report_pressure(*this, pressure); + } + + void notify(resource_provider_base& provider, request_id id) override { + __TBB_ASSERT(&provider == &m_resource_provider, "Provider-consumer mismatch"); + m_body_ptr->notify(id); + tbb::detail::suppress_unused_warning(provider); + } + + void set_body_ptr(resource_limited_body_type* body_ptr) { + m_body_ptr = body_ptr; + } + +private: + ResourceProvider& m_resource_provider; + resource_limited_body_type* m_body_ptr; +}; + +template +struct request_data { + Input input_message; + OutputPorts& output_ports; + std::atomic notify_counter; + HandlesTuple handles; + + request_data(const Input& input, OutputPorts& ports) + : input_message(input) + , output_ports(ports) + , notify_counter(std::tuple_size::value + 1) + {} +}; + +template +struct request_resources_helper { + template + static void run(ConsumerTuple& consumers, request_id id) { + std::get(consumers).request_from_provider(id); + request_resources_helper::run(consumers, id); + } +}; + +template +struct request_resources_helper { + template + static void run(ConsumerTuple&, request_id) {} +}; + +template +struct release_resources_helper { + template + static void run(ConsumerTuple& consumers, request_id id, RequestData& req_data) { + std::get(consumers).release_to_provider(id, std::move(std::get(req_data.handles))); + std::get(req_data.handles) = {}; + release_resources_helper::run(consumers, id, req_data); + } +}; + +template <> +struct release_resources_helper<0> { + template + static void run(ConsumerTuple&, request_id, RequestData&) {} +}; + +template +struct set_body_ptr_helper { + template + static void run(ConsumerTuple& consumers, Body* body_ptr) { + std::get(consumers).set_body_ptr(body_ptr); + set_body_ptr_helper::run(consumers, body_ptr); + } +}; + +template <> +struct set_body_ptr_helper<0> { + template + static void run(ConsumerTuple&, Body*) {} +}; + +template +struct acquire_resources_helper { + template + static bool run(Body* body_ptr, ConsumerTuple& consumers, request_id id, RequestData& req_data) { + __TBB_ASSERT(req_data.notify_counter == 1, "Incorrect notify counter"); + ++req_data.notify_counter; // Local counter in case the resource is denied + auto handle_optional = std::get(consumers).acquire_from_provider(id); + if (handle_optional.has_value()) { + // Successfully acquired resource - save the handle and proceed to the next resource + --req_data.notify_counter; + std::get(req_data.handles) = std::move(handle_optional); + return acquire_resources_helper::run(body_ptr, consumers, id, req_data); + } else { + __TBB_ASSERT(req_data.notify_counter >= 1, "Incorrect notify counter"); + // One of the resources denied the request + release_resources_helper::run(consumers, id, req_data); + body_ptr->release_self_ref(id, req_data); // release the self-reference held at the beginning of resource acquisition + return false; + } + } +}; + +template +struct acquire_resources_helper { + template + static bool run(Body*, ConsumerTuple&, request_id, RequestData& req_data) { + --req_data.notify_counter; + return true; + } +}; + +template +struct report_pressure_helper { + template + static void run(ConsumerTuple& consumers, std::size_t pressure) { + std::get(consumers).report_pressure_to_provider(pressure); + report_pressure_helper::run(consumers, pressure); + } +}; + +template +struct report_pressure_helper { + template + static void run(ConsumerTuple&, std::size_t) {} +}; + +template +class try_acquire_resources_and_execute_task : public graph_task { + BodyLeaf* m_body; + request_id m_id; + RequestDataType& m_request_data; + +public: + try_acquire_resources_and_execute_task(graph& g, d1::small_object_allocator& allocator, BodyLeaf* body_leaf, + request_id id, RequestDataType& request_data) + : graph_task(g, allocator, no_priority) + , m_body(body_leaf) + , m_id(id) + , m_request_data(request_data) + { + __TBB_ASSERT(body_leaf != nullptr, nullptr); + } + + d1::task* execute(d1::execution_data& ed) override { + m_body->try_acquire_resources_and_execute(m_id, m_request_data); + graph_task::template finalize(ed); + return nullptr; + } + + d1::task* cancel(d1::execution_data& ed) override { + m_body->remove_request(m_id); + graph_task::template finalize(ed); + return nullptr; + } +}; + +template +class resource_limited_body_leaf + : public resource_limited_body +{ + using handles_tuple_type = std::tuple; + using consumers_tuple_type = std::tuple...>; + using request_data_type = request_data; + // TODO: should concurrent container be used instead? + using requests_map_type = std::unordered_map; + + tbb::spin_mutex m_mutex; + requests_map_type m_requests; + consumers_tuple_type m_consumers; + Body m_body; + std::uint64_t m_counter; + + template + resource_limited_body_leaf(graph& g, ConsumersTuple&& consumers_tuple, const Body& body) + : resource_limited_body(g) + , m_consumers(std::forward(consumers_tuple)) + , m_body(body) + , m_counter(0) + {} + +public: + resource_limited_body_leaf(graph& g, std::tuple resource_providers, const Body& body) + : resource_limited_body_leaf(g, get_consumers_tuple(resource_providers), body) + {} + + consumers_tuple_type get_consumers_tuple(std::tuple resource_providers) { + return get_consumers_tuple_impl(resource_providers, tbb::detail::make_index_sequence()); + } + + template + consumers_tuple_type get_consumers_tuple_impl(std::tuple resource_providers, + tbb::detail::index_sequence) + { + return consumers_tuple_type({std::get(resource_providers), this}...); + } + + void operator()(const Input& input, OutputPorts& ports) override { + auto& res = form_request(input, ports); + report_pressure(0); // TODO: report real pressure + request_resources(res.first); + release_self_ref(res.first, res.second); + } + + typename requests_map_type::reference form_request(const Input& input, OutputPorts& ports) { + tbb::spin_mutex::scoped_lock lock(m_mutex); + request_id id{++m_counter}; + auto res = m_requests.emplace(std::piecewise_construct, + std::forward_as_tuple(id), + std::forward_as_tuple(input, ports)); + this->graph_reference().reserve_wait(); + __TBB_ASSERT(res.second, "Duplicated requests in the map"); + return *res.first; + } + + void request_resources(request_id id) { + request_resources_helper<0, sizeof...(ResourceProviders)>::run(m_consumers, id); + } + + void release_self_ref(request_id id, request_data_type& req_data) { + std::size_t prev_value = req_data.notify_counter--; + __TBB_ASSERT(prev_value != 0, "Overflow detected"); + if (prev_value == 1) { + try_acquire_resources_and_execute(id, req_data); + } + } + + bool try_acquire_resources(request_id id, request_data_type& req_data) { + // Increment the counter to avoid another resource reacquisition by notify() + // while current acquisition is in progress + std::size_t prev_value = req_data.notify_counter++; + __TBB_ASSERT(prev_value == 0, "Incorrect notify counter before acquisition"); + tbb::detail::suppress_unused_warning(prev_value); + return acquire_resources_helper<0, sizeof...(ResourceProviders)>::run(this, m_consumers, id, req_data); + } + + void try_acquire_resources_and_execute(request_id id, request_data_type& req_data) { + if (try_acquire_resources(id, req_data)) { + // Access to all resources is granted + try_call([&] { + call_body(req_data.input_message, req_data.output_ports, req_data.handles); + }).on_completion([&] { + release_resources(id, req_data); + remove_request(id); + }); + } + } + + void release_resources(request_id id, request_data_type& req_data) { + // TODO: report real pressure, investigate if it should be done before or after the release + report_pressure(0); + release_resources_helper::run(m_consumers, id, req_data); + } + + void remove_request(request_id id) { + tbb::spin_mutex::scoped_lock lock(m_mutex); + std::size_t num_removed = m_requests.erase(id); + this->graph_reference().release_wait(); + __TBB_ASSERT(num_removed == 1, "Removing unregistered request"); + tbb::detail::suppress_unused_warning(num_removed); + } + + void notify(request_id id) override { + tbb::spin_mutex::scoped_lock lock(m_mutex); + auto res = m_requests.find(id); + __TBB_ASSERT(res != m_requests.end(), "Cannot find request for notification"); + request_data_type& data = res->second; + lock.release(); + + std::size_t prev_value = data.notify_counter--; + __TBB_ASSERT(prev_value != 0, "Overflow detected"); + if (prev_value == 1) { + d1::small_object_allocator allocator; + using task_type = try_acquire_resources_and_execute_task; + graph_task* t = allocator.new_object(this->graph_reference(), allocator, this, id, data); + spawn_in_graph_arena(this->graph_reference(), *t); + } + } + + void report_pressure(std::size_t pressure) { + report_pressure_helper<0, sizeof...(ResourceProviders)>::run(m_consumers, pressure); + } + + resource_limited_body_leaf* clone() override { + resource_limited_body_leaf* new_body = new resource_limited_body_leaf(this->graph_reference(), m_consumers, this->m_body); + set_body_ptr_helper::run(new_body->m_consumers, new_body); + return new_body; + } + + void* get_body_ptr() override { return &m_body; } + + template + void call_body(const Input& input, OutputPorts& ports, ResourceHandlesTuple& tuple) { + call_body_impl(input, ports, tuple, + tbb::detail::make_index_sequence::value>()); + } + + template + void call_body_impl(const Input& input, OutputPorts& ports, ResourceHandlesTuple& tuple, + tbb::detail::index_sequence) { + tbb::detail::invoke(m_body, input, ports, std::get(tuple).value()...); + } +}; + +template +class resource_limited_input + : public function_input_base, + resource_limited_input> +{ +public: + static constexpr int N = std::tuple_size::value; + using input_type = Input; + using output_ports_type = OutputPorts; + using resource_limited_body_type = resource_limited_body; + using class_type = resource_limited_input; + using base_type = function_input_base, class_type>; + using input_queue_type = function_input_queue>; + + template + resource_limited_input(graph& g, std::size_t max_concurrency, + std::tuple resource_providers, + Body& body) + : base_type(g, max_concurrency, no_priority, is_body_noexcept(body, resource_providers)) + , m_body(new resource_limited_body_leaf(g, resource_providers, body)) + , m_init_body(new resource_limited_body_leaf(g, resource_providers, body)) + , m_output_ports(init_output_ports::call(g, m_output_ports)) + {} + + resource_limited_input(const resource_limited_input& other) + : base_type(other) + , m_body(other.m_init_body->clone()) + , m_init_body(other.m_init_body->clone()) + , m_output_ports(init_output_ports::call(this->graph_reference(), m_output_ports)) + {} + + ~resource_limited_input() { + delete m_body; + delete m_init_body; + } + + graph_task* apply_body_impl_bypass(const input_type& i + __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo&)) + { + (*m_body)(i, m_output_ports); + graph_task* ttask = nullptr; + if (base_type::my_max_concurrency != 0) { + ttask = base_type::try_get_postponed_task(i); + } + return ttask ? ttask : SUCCESSFULLY_ENQUEUED; + } + + output_ports_type& output_ports() { return m_output_ports; } + + template + Body copy_function_object() { + return *static_cast(m_body->get_body_ptr()); + } +protected: + void reset(reset_flags f) { + base_type::reset_function_input_base(f); + if (f & rf_clear_edges) clear_element::clear_this(m_output_ports); + if (f & rf_reset_bodies) { + resource_limited_body_type* tmp = m_init_body->clone(); + delete m_body; + m_body = tmp; + } + __TBB_ASSERT(!(f & rf_clear_edges) || clear_element::this_empty(m_output_ports), "resource_limited_node reset failed"); + } +private: + template + bool is_body_noexcept(Body& body, std::tuple) { + return noexcept(tbb::detail::invoke(body, std::declval(), m_output_ports, + std::declval()...)); + } + + resource_limited_body_type* m_body; + resource_limited_body_type* m_init_body; + output_ports_type m_output_ports; +}; + +template +class resource_limited_node + : public graph_node + , public resource_limited_input::type> +{ +public: + using input_type = Input; + using output_type = null_type; + using output_ports_type = typename wrap_tuple_elements::type; +private: + using input_impl_type = resource_limited_input; + using input_impl_type::my_predecessors; +public: + template + resource_limited_node(graph& g, std::size_t concurrency, + std::tuple resource_providers, + Body body) + : graph_node(g) + , input_impl_type(g, concurrency, resource_providers, body) + {} + + resource_limited_node(const resource_limited_node& other) + : graph_node(other.my_graph) + , input_impl_type(other) + {} +protected: + void reset_node(reset_flags f) override { input_impl_type::reset(f); } +}; // class resource_limited_node + +} // namespace d2 +} // namespace detail +} // namespace tbb + +#endif // __TBB__flow_graph_resource_limiting_H diff --git a/src/3rd party/oneapi/tbb/detail/_flow_graph_tagged_buffer_impl.h b/src/3rd party/oneapi/tbb/detail/_flow_graph_tagged_buffer_impl.h new file mode 100644 index 0000000000..17acd5c361 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_flow_graph_tagged_buffer_impl.h @@ -0,0 +1,380 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// a hash table buffer that can expand, and can support as many deletions as +// additions, list-based, with elements of list held in array (for destruction +// management), multiplicative hashing (like ets). No synchronization built-in. +// + +#ifndef __TBB__flow_graph_hash_buffer_impl_H +#define __TBB__flow_graph_hash_buffer_impl_H + +#ifndef __TBB_flow_graph_H +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +// included in namespace tbb::flow::interfaceX::internal + +// elements in the table are a simple list; we need pointer to next element to +// traverse the chain + +template +struct hash_buffer_element : public aligned_pair { + using key_type = Key; + using value_type = ValueType; + + value_type* get_value_ptr() { return reinterpret_cast(this->first); } + hash_buffer_element* get_next() { return reinterpret_cast(this->second); } + void set_next(hash_buffer_element* new_next) { this->second = reinterpret_cast(new_next); } + + void create_element(const value_type& v) { + ::new(this->first) value_type(v); + } + + void create_element(hash_buffer_element&& other) { + ::new(this->first) value_type(std::move(*other.get_value_ptr())); + } + + void destroy_element() { + get_value_ptr()->~value_type(); + } +}; + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +template +struct metainfo_hash_buffer_element : public aligned_triple { + using key_type = Key; + using value_type = ValueType; + + value_type* get_value_ptr() { return reinterpret_cast(this->first); } + metainfo_hash_buffer_element* get_next() { + return reinterpret_cast(this->second); + } + void set_next(metainfo_hash_buffer_element* new_next) { this->second = reinterpret_cast(new_next); } + message_metainfo& get_metainfo() { return this->third; } + + void create_element(const value_type& v, const message_metainfo& metainfo) { + __TBB_ASSERT(this->third.empty(), nullptr); + ::new(this->first) value_type(v); + this->third = metainfo; + + for (auto waiter : metainfo.waiters()) { + waiter->reserve(1); + } + } + + void create_element(metainfo_hash_buffer_element&& other) { + __TBB_ASSERT(this->third.empty(), nullptr); + ::new(this->first) value_type(std::move(*other.get_value_ptr())); + this->third = std::move(other.get_metainfo()); + } + + void destroy_element() { + get_value_ptr()->~value_type(); + + for (auto waiter : get_metainfo().waiters()) { + waiter->release(1); + } + get_metainfo() = message_metainfo{}; + } +}; +#endif + +template + < + typename ElementType, + typename ValueToKey, // abstract method that returns "const Key" or "const Key&" given ValueType + typename HashCompare, // has hash and equal + typename Allocator=tbb::cache_aligned_allocator + > +class hash_buffer_impl : public HashCompare { +public: + static const size_t INITIAL_SIZE = 8; // initial size of the hash pointer table + typedef typename ElementType::key_type key_type; + typedef typename ElementType::value_type value_type; + typedef ElementType element_type; + typedef value_type *pointer_type; + typedef element_type *list_array_type; // array we manage manually + typedef list_array_type *pointer_array_type; + typedef typename std::allocator_traits::template rebind_alloc pointer_array_allocator_type; + typedef typename std::allocator_traits::template rebind_alloc elements_array_allocator; + typedef typename std::decay::type Knoref; + +private: + ValueToKey *my_key; + size_t my_size; + size_t nelements; + pointer_array_type pointer_array; // pointer_array[my_size] + list_array_type elements_array; // elements_array[my_size / 2] + element_type* free_list; + + size_t mask() { return my_size - 1; } + + void set_up_free_list( element_type **p_free_list, list_array_type la, size_t sz) { + for(size_t i=0; i < sz - 1; ++i ) { // construct free list + la[i].set_next(&(la[i + 1])); + } + la[sz - 1].set_next(nullptr); + *p_free_list = (element_type *)&(la[0]); + } + + // cleanup for exceptions + struct DoCleanup { + pointer_array_type *my_pa; + list_array_type *my_elements; + size_t my_size; + + DoCleanup(pointer_array_type &pa, list_array_type &my_els, size_t sz) : + my_pa(&pa), my_elements(&my_els), my_size(sz) { } + ~DoCleanup() { + if(my_pa) { + size_t dont_care = 0; + internal_free_buffer(*my_pa, *my_elements, my_size, dont_care); + } + } + }; + + // exception-safety requires we do all the potentially-throwing operations first + void grow_array() { + size_t new_size = my_size*2; + size_t new_nelements = nelements; // internal_free_buffer zeroes this + list_array_type new_elements_array = nullptr; + pointer_array_type new_pointer_array = nullptr; + list_array_type new_free_list = nullptr; + { + DoCleanup my_cleanup(new_pointer_array, new_elements_array, new_size); + new_elements_array = elements_array_allocator().allocate(my_size); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + for (std::size_t i = 0; i < my_size; ++i) { + ::new(new_elements_array + i) element_type(); + } +#endif + new_pointer_array = pointer_array_allocator_type().allocate(new_size); + for(size_t i=0; i < new_size; ++i) new_pointer_array[i] = nullptr; + set_up_free_list(&new_free_list, new_elements_array, my_size ); + + for(size_t i=0; i < my_size; ++i) { + for( element_type* op = pointer_array[i]; op; op = (element_type *)(op->get_next())) { + internal_insert_with_key(new_pointer_array, new_size, new_free_list, std::move(*op)); + } + } + my_cleanup.my_pa = nullptr; + my_cleanup.my_elements = nullptr; + } + + internal_free_buffer(pointer_array, elements_array, my_size, nelements); + free_list = new_free_list; + pointer_array = new_pointer_array; + elements_array = new_elements_array; + my_size = new_size; + nelements = new_nelements; + } + + // v should have perfect forwarding if std::move implemented. + // we use this method to move elements in grow_array, so can't use class fields + template + const value_type& get_value_from_pack(const Value& value, const Args&...) { + return value; + } + + template + const value_type& get_value_from_pack(Element&& element) { + return *(element.get_value_ptr()); + } + + template + void internal_insert_with_key( element_type **p_pointer_array, size_t p_sz, list_array_type &p_free_list, + Args&&... args) { + size_t l_mask = p_sz-1; + __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); + size_t h = this->hash(tbb::detail::invoke(*my_key, get_value_from_pack(args...))) & l_mask; + __TBB_ASSERT(p_free_list, "Error: free list not set up."); + element_type* my_elem = p_free_list; p_free_list = (element_type *)(p_free_list->get_next()); + my_elem->create_element(std::forward(args)...); + my_elem->set_next(p_pointer_array[h]); + p_pointer_array[h] = my_elem; + } + + void internal_initialize_buffer() { + pointer_array = pointer_array_allocator_type().allocate(my_size); + for(size_t i = 0; i < my_size; ++i) pointer_array[i] = nullptr; + elements_array = elements_array_allocator().allocate(my_size / 2); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + for (std::size_t i = 0; i < my_size / 2; ++i) { + ::new(elements_array + i) element_type(); + } +#endif + set_up_free_list(&free_list, elements_array, my_size / 2); + } + + // made static so an enclosed class can use to properly dispose of the internals + static void internal_free_buffer( pointer_array_type &pa, list_array_type &el, size_t &sz, size_t &ne ) { + if(pa) { + for(size_t i = 0; i < sz; ++i ) { + element_type *p_next; + for( element_type *p = pa[i]; p; p = p_next) { + p_next = p->get_next(); + p->destroy_element(); + } + } + pointer_array_allocator_type().deallocate(pa, sz); + pa = nullptr; + } + // Separate test (if allocation of pa throws, el may be allocated. + // but no elements will be constructed.) + if(el) { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + for (std::size_t i = 0; i < sz / 2; ++i) { + (el + i)->~element_type(); + } +#endif + elements_array_allocator().deallocate(el, sz / 2); + el = nullptr; + } + sz = INITIAL_SIZE; + ne = 0; + } + +public: + hash_buffer_impl() : my_key(nullptr), my_size(INITIAL_SIZE), nelements(0) { + internal_initialize_buffer(); + } + + ~hash_buffer_impl() { + internal_free_buffer(pointer_array, elements_array, my_size, nelements); + delete my_key; + my_key = nullptr; + } + hash_buffer_impl(const hash_buffer_impl&) = delete; + hash_buffer_impl& operator=(const hash_buffer_impl&) = delete; + + void reset() { + internal_free_buffer(pointer_array, elements_array, my_size, nelements); + internal_initialize_buffer(); + } + + // Take ownership of func object allocated with new. + // This method is only used internally, so can't be misused by user. + void set_key_func(ValueToKey *vtk) { my_key = vtk; } + // pointer is used to clone() + ValueToKey* get_key_func() { return my_key; } + + template + bool insert_with_key(const value_type &v, Args&&... args) { + element_type* p = nullptr; + __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); + if(find_element_ref_with_key(tbb::detail::invoke(*my_key, v), p)) { + p->destroy_element(); + p->create_element(v, std::forward(args)...); + return false; + } + ++nelements; + if(nelements*2 > my_size) grow_array(); + internal_insert_with_key(pointer_array, my_size, free_list, v, std::forward(args)...); + return true; + } + + bool find_element_ref_with_key(const Knoref& k, element_type*& v) { + size_t i = this->hash(k) & mask(); + for(element_type* p = pointer_array[i]; p; p = (element_type *)(p->get_next())) { + __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); + if(this->equal(tbb::detail::invoke(*my_key, *p->get_value_ptr()), k)) { + v = p; + return true; + } + } + return false; + } + + // returns true and sets v to array element if found, else returns false. + bool find_ref_with_key(const Knoref& k, pointer_type &v) { + element_type* element_ptr = nullptr; + bool res = find_element_ref_with_key(k, element_ptr); + if(res) v = element_ptr->get_value_ptr(); + return res; + } + + bool find_with_key( const Knoref& k, value_type &v) { + value_type *p; + if(find_ref_with_key(k, p)) { + v = *p; + return true; + } + else + return false; + } + + void delete_with_key(const Knoref& k) { + size_t h = this->hash(k) & mask(); + element_type* prev = nullptr; + for(element_type* p = pointer_array[h]; p; prev = p, p = (element_type *)(p->get_next())) { + value_type *vp = p->get_value_ptr(); + __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); + if(this->equal(tbb::detail::invoke(*my_key, *vp), k)) { + p->destroy_element(); + if(prev) prev->set_next(p->get_next()); + else pointer_array[h] = (element_type *)(p->get_next()); + p->set_next(free_list); + free_list = p; + --nelements; + return; + } + } + __TBB_ASSERT(false, "key not found for delete"); + } +}; + +template + < + typename Key, // type of key within ValueType + typename ValueType, + typename ValueToKey, // abstract method that returns "const Key" or "const Key&" given ValueType + typename HashCompare, // has hash and equal + typename Allocator=tbb::cache_aligned_allocator> + > +using hash_buffer = hash_buffer_impl, + ValueToKey, HashCompare, Allocator>; + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +template + < + typename Key, // type of key within ValueType + typename ValueType, + typename ValueToKey, // abstract method that returns "const Key" or "const Key&" given ValueType + typename HashCompare, // has hash and equal + typename Allocator=tbb::cache_aligned_allocator> + > +struct metainfo_hash_buffer : public hash_buffer_impl, + ValueToKey, HashCompare, Allocator> +{ +private: + using base_type = hash_buffer_impl, + ValueToKey, HashCompare, Allocator>; +public: + bool find_with_key(const typename base_type::Knoref& k, + typename base_type::value_type& v, message_metainfo& metainfo) + { + typename base_type::element_type* p = nullptr; + bool result = this->find_element_ref_with_key(k, p); + if (result) { + v = *(p->get_value_ptr()); + metainfo = p->get_metainfo(); + } + return result; + } +}; +#endif // __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +#endif // __TBB__flow_graph_hash_buffer_impl_H diff --git a/src/3rd party/oneapi/tbb/detail/_flow_graph_trace_impl.h b/src/3rd party/oneapi/tbb/detail/_flow_graph_trace_impl.h new file mode 100644 index 0000000000..07a2d98a2c --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_flow_graph_trace_impl.h @@ -0,0 +1,365 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _FGT_GRAPH_TRACE_IMPL_H +#define _FGT_GRAPH_TRACE_IMPL_H + +#include "../profiling.h" +#if (_MSC_VER >= 1900) + #include +#endif + +namespace tbb { +namespace detail { +namespace d2 { + +template< typename T > class sender; +template< typename T > class receiver; + +#if TBB_USE_PROFILING_TOOLS + #if __TBB_FLOW_TRACE_CODEPTR + #if (_MSC_VER >= 1900) + #define CODEPTR() (_ReturnAddress()) + #elif __TBB_GCC_VERSION >= 40800 + #define CODEPTR() ( __builtin_return_address(0)) + #else + #define CODEPTR() nullptr + #endif + #else + #define CODEPTR() nullptr + #endif /* __TBB_FLOW_TRACE_CODEPTR */ + +inline void fgt_alias_port(void *node, void *p, bool visible) { + if(visible) + itt_relation_add( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_NODE ); + else + itt_relation_add( d1::ITT_DOMAIN_FLOW, p, FLOW_NODE, __itt_relation_is_child_of, node, FLOW_NODE ); +} + +inline void fgt_composite ( void* codeptr, void *node, void *graph ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, graph, FLOW_GRAPH, FLOW_COMPOSITE_NODE ); + suppress_unused_warning( codeptr ); +#if __TBB_FLOW_TRACE_CODEPTR + if (codeptr != nullptr) { + register_node_addr(d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, CODE_ADDRESS, &codeptr); + } +#endif +} + +inline void fgt_internal_alias_input_port( void *node, void *p, string_resource_index name_index ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, p, FLOW_INPUT_PORT, node, FLOW_NODE, name_index ); + itt_relation_add( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_INPUT_PORT ); +} + +inline void fgt_internal_alias_output_port( void *node, void *p, string_resource_index name_index ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, p, FLOW_OUTPUT_PORT, node, FLOW_NODE, name_index ); + itt_relation_add( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_OUTPUT_PORT ); +} + +template +void alias_input_port(void *node, receiver* port, string_resource_index name_index) { + // TODO: Make fgt_internal_alias_input_port a function template? + fgt_internal_alias_input_port( node, port, name_index); +} + +template < typename PortsTuple, int N > +struct fgt_internal_input_alias_helper { + static void alias_port( void *node, PortsTuple &ports ) { + alias_input_port( node, &(std::get(ports)), static_cast(FLOW_INPUT_PORT_0 + N - 1) ); + fgt_internal_input_alias_helper::alias_port( node, ports ); + } +}; + +template < typename PortsTuple > +struct fgt_internal_input_alias_helper { + static void alias_port( void * /* node */, PortsTuple & /* ports */ ) { } +}; + +template +void alias_output_port(void *node, sender* port, string_resource_index name_index) { + // TODO: Make fgt_internal_alias_output_port a function template? + fgt_internal_alias_output_port( node, static_cast(port), name_index); +} + +template < typename PortsTuple, int N > +struct fgt_internal_output_alias_helper { + static void alias_port( void *node, PortsTuple &ports ) { + alias_output_port( node, &(std::get(ports)), static_cast(FLOW_OUTPUT_PORT_0 + N - 1) ); + fgt_internal_output_alias_helper::alias_port( node, ports ); + } +}; + +template < typename PortsTuple > +struct fgt_internal_output_alias_helper { + static void alias_port( void * /*node*/, PortsTuple &/*ports*/ ) { + } +}; + +inline void fgt_internal_create_input_port( void *node, void *p, string_resource_index name_index ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, p, FLOW_INPUT_PORT, node, FLOW_NODE, name_index ); +} + +inline void fgt_internal_create_output_port( void* codeptr, void *node, void *p, string_resource_index name_index ) { + itt_make_task_group(d1::ITT_DOMAIN_FLOW, p, FLOW_OUTPUT_PORT, node, FLOW_NODE, name_index); + suppress_unused_warning( codeptr ); +#if __TBB_FLOW_TRACE_CODEPTR + if (codeptr != nullptr) { + register_node_addr(d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, CODE_ADDRESS, &codeptr); + } +#endif +} + +template +void register_input_port(void *node, receiver* port, string_resource_index name_index) { + // TODO: Make fgt_internal_create_input_port a function template? + fgt_internal_create_input_port(node, static_cast(port), name_index); +} + +template < typename PortsTuple, int N > +struct fgt_internal_input_helper { + static void register_port( void *node, PortsTuple &ports ) { + register_input_port( node, &(std::get(ports)), static_cast(FLOW_INPUT_PORT_0 + N - 1) ); + fgt_internal_input_helper::register_port( node, ports ); + } +}; + +template < typename PortsTuple > +struct fgt_internal_input_helper { + static void register_port( void *node, PortsTuple &ports ) { + register_input_port( node, &(std::get<0>(ports)), FLOW_INPUT_PORT_0 ); + } +}; + +template +void register_output_port(void* codeptr, void *node, sender* port, string_resource_index name_index) { + // TODO: Make fgt_internal_create_output_port a function template? + fgt_internal_create_output_port( codeptr, node, static_cast(port), name_index); +} + +template < typename PortsTuple, int N > +struct fgt_internal_output_helper { + static void register_port( void* codeptr, void *node, PortsTuple &ports ) { + register_output_port( codeptr, node, &(std::get(ports)), static_cast(FLOW_OUTPUT_PORT_0 + N - 1) ); + fgt_internal_output_helper::register_port( codeptr, node, ports ); + } +}; + +template < typename PortsTuple > +struct fgt_internal_output_helper { + static void register_port( void* codeptr, void *node, PortsTuple &ports ) { + register_output_port( codeptr, node, &(std::get<0>(ports)), FLOW_OUTPUT_PORT_0 ); + } +}; + +template< typename NodeType > +void fgt_multioutput_node_desc( const NodeType *node, const char *desc ) { + void *addr = (void *)( static_cast< receiver< typename NodeType::input_type > * >(const_cast< NodeType *>(node)) ); + itt_metadata_str_add( d1::ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc ); +} + +template< typename NodeType > +void fgt_multiinput_multioutput_node_desc( const NodeType *node, const char *desc ) { + void *addr = const_cast(node); + itt_metadata_str_add( d1::ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc ); +} + +template< typename NodeType > +inline void fgt_node_desc( const NodeType *node, const char *desc ) { + void *addr = (void *)( static_cast< sender< typename NodeType::output_type > * >(const_cast< NodeType *>(node)) ); + itt_metadata_str_add( d1::ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc ); +} + +inline void fgt_graph_desc( const void *g, const char *desc ) { + void *addr = const_cast< void *>(g); + itt_metadata_str_add( d1::ITT_DOMAIN_FLOW, addr, FLOW_GRAPH, FLOW_OBJECT_NAME, desc ); +} + +inline void fgt_body( void *node, void *body ) { + itt_relation_add( d1::ITT_DOMAIN_FLOW, body, FLOW_BODY, __itt_relation_is_child_of, node, FLOW_NODE ); +} + +template< int N, typename PortsTuple > +inline void fgt_multioutput_node(void* codeptr, string_resource_index t, void *g, void *input_port, PortsTuple &ports ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, input_port, FLOW_NODE, g, FLOW_GRAPH, t ); + fgt_internal_create_input_port( input_port, input_port, FLOW_INPUT_PORT_0 ); + fgt_internal_output_helper::register_port(codeptr, input_port, ports ); +} + +template< int N, typename PortsTuple > +inline void fgt_multioutput_node_with_body( void* codeptr, string_resource_index t, void *g, void *input_port, PortsTuple &ports, void *body ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, input_port, FLOW_NODE, g, FLOW_GRAPH, t ); + fgt_internal_create_input_port( input_port, input_port, FLOW_INPUT_PORT_0 ); + fgt_internal_output_helper::register_port( codeptr, input_port, ports ); + fgt_body( input_port, body ); +} + +template< int N, typename PortsTuple > +inline void fgt_multiinput_node( void* codeptr, string_resource_index t, void *g, PortsTuple &ports, void *output_port) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t ); + fgt_internal_create_output_port( codeptr, output_port, output_port, FLOW_OUTPUT_PORT_0 ); + fgt_internal_input_helper::register_port( output_port, ports ); +} + +inline void fgt_multiinput_multioutput_node( void* codeptr, string_resource_index t, void *n, void *g ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, n, FLOW_NODE, g, FLOW_GRAPH, t ); + suppress_unused_warning( codeptr ); +#if __TBB_FLOW_TRACE_CODEPTR + if (codeptr != nullptr) { + register_node_addr(d1::ITT_DOMAIN_FLOW, n, FLOW_NODE, CODE_ADDRESS, &codeptr); + } +#endif +} + +inline void fgt_node( void* codeptr, string_resource_index t, void *g, void *output_port ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t ); + fgt_internal_create_output_port( codeptr, output_port, output_port, FLOW_OUTPUT_PORT_0 ); +} + +inline void fgt_node_with_body( void* codeptr, string_resource_index t, void *g, void *output_port, void *body ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t ); + fgt_internal_create_output_port(codeptr, output_port, output_port, FLOW_OUTPUT_PORT_0 ); + fgt_body( output_port, body ); +} + +inline void fgt_node( void* codeptr, string_resource_index t, void *g, void *input_port, void *output_port ) { + fgt_node( codeptr, t, g, output_port ); + fgt_internal_create_input_port( output_port, input_port, FLOW_INPUT_PORT_0 ); +} + +inline void fgt_node_with_body( void* codeptr, string_resource_index t, void *g, void *input_port, void *output_port, void *body ) { + fgt_node_with_body( codeptr, t, g, output_port, body ); + fgt_internal_create_input_port( output_port, input_port, FLOW_INPUT_PORT_0 ); +} + + +inline void fgt_node( void* codeptr, string_resource_index t, void *g, void *input_port, void *decrement_port, void *output_port ) { + fgt_node( codeptr, t, g, input_port, output_port ); + fgt_internal_create_input_port( output_port, decrement_port, FLOW_INPUT_PORT_1 ); +} + +inline void fgt_make_edge( void *output_port, void *input_port ) { + itt_relation_add( d1::ITT_DOMAIN_FLOW, output_port, FLOW_OUTPUT_PORT, __itt_relation_is_predecessor_to, input_port, FLOW_INPUT_PORT); +} + +inline void fgt_remove_edge( void *output_port, void *input_port ) { + itt_relation_add( d1::ITT_DOMAIN_FLOW, output_port, FLOW_OUTPUT_PORT, __itt_relation_is_sibling_of, input_port, FLOW_INPUT_PORT); +} + +inline void fgt_graph( void *g ) { + itt_make_task_group( d1::ITT_DOMAIN_FLOW, g, FLOW_GRAPH, nullptr, FLOW_NULL, FLOW_GRAPH ); +} + +inline void fgt_begin_body( void *body ) { + itt_task_begin( d1::ITT_DOMAIN_FLOW, body, FLOW_BODY, nullptr, FLOW_NULL, FLOW_BODY ); +} + +inline void fgt_end_body( void * ) { + itt_task_end( d1::ITT_DOMAIN_FLOW ); +} + +inline void fgt_async_try_put_begin( void *node, void *port ) { + itt_task_begin( d1::ITT_DOMAIN_FLOW, port, FLOW_OUTPUT_PORT, node, FLOW_NODE, FLOW_OUTPUT_PORT ); +} + +inline void fgt_async_try_put_end( void *, void * ) { + itt_task_end( d1::ITT_DOMAIN_FLOW ); +} + +inline void fgt_async_reserve( void *node, void *graph ) { + itt_region_begin( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE, graph, FLOW_GRAPH, FLOW_NULL ); +} + +inline void fgt_async_commit( void *node, void * /*graph*/) { + itt_region_end( d1::ITT_DOMAIN_FLOW, node, FLOW_NODE ); +} + +inline void fgt_reserve_wait( void *graph ) { + itt_region_begin( d1::ITT_DOMAIN_FLOW, graph, FLOW_GRAPH, nullptr, FLOW_NULL, FLOW_NULL ); +} + +inline void fgt_release_wait( void *graph ) { + itt_region_end( d1::ITT_DOMAIN_FLOW, graph, FLOW_GRAPH ); +} + +#else // TBB_USE_PROFILING_TOOLS + +#define CODEPTR() nullptr + +inline void fgt_alias_port(void * /*node*/, void * /*p*/, bool /*visible*/ ) { } + +inline void fgt_composite ( void* /*codeptr*/, void * /*node*/, void * /*graph*/ ) { } + +inline void fgt_graph( void * /*g*/ ) { } + +template< typename NodeType > +inline void fgt_multioutput_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { } + +template< typename NodeType > +inline void fgt_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { } + +inline void fgt_graph_desc( const void * /*g*/, const char * /*desc*/ ) { } + +template< int N, typename PortsTuple > +inline void fgt_multioutput_node( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*input_port*/, PortsTuple & /*ports*/ ) { } + +template< int N, typename PortsTuple > +inline void fgt_multioutput_node_with_body( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*input_port*/, PortsTuple & /*ports*/, void * /*body*/ ) { } + +template< int N, typename PortsTuple > +inline void fgt_multiinput_node( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, PortsTuple & /*ports*/, void * /*output_port*/ ) { } + +inline void fgt_multiinput_multioutput_node( void* /*codeptr*/, string_resource_index /*t*/, void * /*node*/, void * /*graph*/ ) { } + +inline void fgt_node( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*output_port*/ ) { } +inline void fgt_node( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*decrement_port*/, void * /*output_port*/ ) { } + +inline void fgt_node_with_body( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*output_port*/, void * /*body*/ ) { } +inline void fgt_node_with_body( void* /*codeptr*/, string_resource_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*output_port*/, void * /*body*/ ) { } + +inline void fgt_make_edge( void * /*output_port*/, void * /*input_port*/ ) { } +inline void fgt_remove_edge( void * /*output_port*/, void * /*input_port*/ ) { } + +inline void fgt_begin_body( void * /*body*/ ) { } +inline void fgt_end_body( void * /*body*/) { } + +inline void fgt_async_try_put_begin( void * /*node*/, void * /*port*/ ) { } +inline void fgt_async_try_put_end( void * /*node*/ , void * /*port*/ ) { } +inline void fgt_async_reserve( void * /*node*/, void * /*graph*/ ) { } +inline void fgt_async_commit( void * /*node*/, void * /*graph*/ ) { } +inline void fgt_reserve_wait( void * /*graph*/ ) { } +inline void fgt_release_wait( void * /*graph*/ ) { } + +template< typename NodeType > +void fgt_multiinput_multioutput_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { } + +template < typename PortsTuple, int N > +struct fgt_internal_input_alias_helper { + static void alias_port( void * /*node*/, PortsTuple & /*ports*/ ) { } +}; + +template < typename PortsTuple, int N > +struct fgt_internal_output_alias_helper { + static void alias_port( void * /*node*/, PortsTuple & /*ports*/ ) { } +}; + +#endif // TBB_USE_PROFILING_TOOLS + +} // d2 +} // namespace detail +} // namespace tbb + +#endif // _FGT_GRAPH_TRACE_IMPL_H diff --git a/src/3rd party/oneapi/tbb/detail/_flow_graph_types_impl.h b/src/3rd party/oneapi/tbb/detail/_flow_graph_types_impl.h new file mode 100644 index 0000000000..df544027e1 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_flow_graph_types_impl.h @@ -0,0 +1,402 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + Copyright (c) 2025 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__flow_graph_types_impl_H +#define __TBB__flow_graph_types_impl_H + +#ifndef __TBB_flow_graph_H +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +// included in namespace tbb::detail::d2 + +// the change to key_matching (adding a K and KHash template parameter, making it a class) +// means we have to pass this data to the key_matching_port. All the ports have only one +// template parameter, so we have to wrap the following types in a trait: +// +// . K == key_type +// . KHash == hash and compare for Key +// . TtoK == function_body that given an object of T, returns its K +// . T == type accepted by port, and stored in the hash table +// +// The port will have an additional parameter on node construction, which is a function_body +// that accepts a const T& and returns a K which is the field in T which is its K. +template +struct KeyTrait { + typedef Kp K; + typedef Tp T; + typedef type_to_key_function_body TtoK; + typedef KHashp KHash; +}; + +// wrap each element of a tuple in a template, and make a tuple of the result. +template class PortType, typename TypeTuple> +struct wrap_tuple_elements; + +// A wrapper that generates the traits needed for each port of a key-matching join, +// and the type of the tuple of input ports. +template class PortType, typename KeyTraits, typename TypeTuple> +struct wrap_key_tuple_elements; + +template class PortType, typename... Args> +struct wrap_tuple_elements >{ + using type = std::tuple...>; +}; + +template class PortType, typename KeyTraits, typename... Args> +struct wrap_key_tuple_elements > { + using key_type = typename KeyTraits::key_type; + using hash_compare_type = typename KeyTraits::hash_compare_type; + using type = std::tuple>...>; +}; + +template< int... S > class sequence {}; + +template< int N, int... S > +struct make_sequence : make_sequence < N - 1, N - 1, S... > {}; + +template< int... S > +struct make_sequence < 0, S... > { + typedef sequence type; +}; + +template struct alignment_of { + typedef struct { char t; U padded; } test_alignment; + static const size_t value = sizeof(test_alignment) - sizeof(U); +}; + +template +struct max_alignment_helper; + +template +struct max_alignment_helper { + using type = typename max_alignment_helper::type>::type; +}; + +template +struct max_alignment_helper { + using type = typename std::conditional::type; +}; + +template +using max_alignment_helper_t = typename max_alignment_helper::type; + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(push) +#pragma warning(disable: 4324) // warning C4324: structure was padded due to alignment specifier +#endif + +// T1, T2 are actual types stored. The space defined for T1 in the type returned +// is a char array of the correct size. Type T2 should be trivially-constructible, +// T1 must be explicitly managed. + +template +struct alignas(alignof(max_alignment_helper_t)) aligned_pair { + char first[sizeof(T1)]; + T2 second; +}; + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +template +struct alignas(alignof(max_alignment_helper_t)) aligned_triple { + char first[sizeof(T1)]; + T2 second; + T3 third; +}; +#endif + + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(pop) // warning 4324 is back +#endif + +// support for variant type +// type we use when we're not storing a value +struct default_constructed { }; + +// type which contains another type, tests for what type is contained, and references to it. +// Wrapper +// void CopyTo( void *newSpace) : builds a Wrapper copy of itself in newSpace + +// struct to allow us to copy and test the type of objects +struct WrapperBase { + virtual ~WrapperBase() {} + virtual void CopyTo(void* /*newSpace*/) const = 0; +}; + +// Wrapper contains a T, with the ability to test what T is. The Wrapper can be +// constructed from a T, can be copy-constructed from another Wrapper, and can be +// examined via value(), but not modified. +template +struct Wrapper: public WrapperBase { + typedef T value_type; + typedef T* pointer_type; +private: + T value_space; +public: + const value_type &value() const { return value_space; } + +private: + Wrapper(); + + // on exception will ensure the Wrapper will contain only a trivially-constructed object + struct _unwind_space { + pointer_type space; + _unwind_space(pointer_type p) : space(p) {} + ~_unwind_space() { + if(space) (void) new (space) Wrapper(default_constructed()); + } + }; +public: + explicit Wrapper( const T& other ) : value_space(other) { } + explicit Wrapper(const Wrapper& other) = delete; + + void CopyTo(void* newSpace) const override { + _unwind_space guard((pointer_type)newSpace); + (void) new(newSpace) Wrapper(value_space); + guard.space = nullptr; + } + ~Wrapper() { } +}; + +// specialization for array objects +template +struct Wrapper : public WrapperBase { + typedef T value_type; + typedef T* pointer_type; + // space must be untyped. + typedef T ArrayType[N]; +private: + // The space is not of type T[N] because when copy-constructing, it would be + // default-initialized and then copied to in some fashion, resulting in two + // constructions and one destruction per element. If the type is char[ ], we + // placement new into each element, resulting in one construction per element. + static const size_t space_size = sizeof(ArrayType); + char value_space[space_size]; + + + // on exception will ensure the already-built objects will be destructed + // (the value_space is a char array, so it is already trivially-destructible.) + struct _unwind_class { + pointer_type space; + int already_built; + _unwind_class(pointer_type p) : space(p), already_built(0) {} + ~_unwind_class() { + if(space) { + for(size_t i = already_built; i > 0 ; --i ) space[i-1].~value_type(); + (void) new(space) Wrapper(default_constructed()); + } + } + }; +public: + const ArrayType &value() const { + char *vp = const_cast(value_space); + return reinterpret_cast(*vp); + } + +private: + Wrapper(); +public: + // have to explicitly construct because other decays to a const value_type* + explicit Wrapper(const ArrayType& other) { + _unwind_class guard((pointer_type)value_space); + pointer_type vp = reinterpret_cast(&value_space); + for(size_t i = 0; i < N; ++i ) { + (void) new(vp++) value_type(other[i]); + ++(guard.already_built); + } + guard.space = nullptr; + } + explicit Wrapper(const Wrapper& other) : WrapperBase() { + // we have to do the heavy lifting to copy contents + _unwind_class guard((pointer_type)value_space); + pointer_type dp = reinterpret_cast(value_space); + pointer_type sp = reinterpret_cast(const_cast(other.value_space)); + for(size_t i = 0; i < N; ++i, ++dp, ++sp) { + (void) new(dp) value_type(*sp); + ++(guard.already_built); + } + guard.space = nullptr; + } + + void CopyTo(void* newSpace) const override { + (void) new(newSpace) Wrapper(*this); // exceptions handled in copy constructor + } + + ~Wrapper() { + // have to destroy explicitly in reverse order + pointer_type vp = reinterpret_cast(&value_space); + for(size_t i = N; i > 0 ; --i ) vp[i-1].~value_type(); + } +}; + +// given a tuple, return the type of the element that has the maximum alignment requirement. +// Given a tuple and that type, return the number of elements of the object with the max +// alignment requirement that is at least as big as the largest object in the tuple. + +template struct pick_one; +template struct pick_one { typedef T1 type; }; +template struct pick_one { typedef T2 type; }; + +template< template class Selector, typename T1, typename T2 > +struct pick_max { + typedef typename pick_one< (Selector::value > Selector::value), T1, T2 >::type type; +}; + +template struct size_of { static const int value = sizeof(T); }; + +template< size_t N, class Tuple, template class Selector > struct pick_tuple_max { + typedef typename pick_tuple_max::type LeftMaxType; + typedef typename std::tuple_element::type ThisType; + typedef typename pick_max::type type; +}; + +template< class Tuple, template class Selector > struct pick_tuple_max<0, Tuple, Selector> { + typedef typename std::tuple_element<0, Tuple>::type type; +}; + +// is the specified type included in a tuple? +template +struct is_element_of { + typedef typename std::tuple_element::type T_i; + static const bool value = std::is_same::value || is_element_of::value; +}; + +template +struct is_element_of { + typedef typename std::tuple_element<0, Tuple>::type T_i; + static const bool value = std::is_same::value; +}; + +// allow the construction of types that are listed tuple. If a disallowed type +// construction is written, a method involving this type is created. The +// type has no definition, so a syntax error is generated. +template struct ERROR_Type_Not_allowed_In_Tagged_Msg_Not_Member_Of_Tuple; + +template struct do_if; +template +struct do_if { + static void construct(void *mySpace, const T& x) { + (void) new(mySpace) Wrapper(x); + } +}; +template +struct do_if { + static void construct(void * /*mySpace*/, const T& x) { + // This method is instantiated when the type T does not match any of the + // element types in the Tuple in variant. + ERROR_Type_Not_allowed_In_Tagged_Msg_Not_Member_Of_Tuple::bad_type(x); + } +}; + +// Tuple tells us the allowed types that variant can hold. It determines the alignment of the space in +// Wrapper, and how big Wrapper is. +// +// the object can only be tested for type, and a read-only reference can be fetched by cast_to(). + +using tbb::detail::punned_cast; + +template +class tagged_msg { + using Tuple = std::tuple; + + class variant { + static const size_t N = std::tuple_size::value; + typedef typename pick_tuple_max::type AlignType; + typedef typename pick_tuple_max::type MaxSizeType; + static const size_t MaxNBytes = (sizeof(Wrapper)+sizeof(AlignType)-1); + static const size_t MaxNElements = MaxNBytes/sizeof(AlignType); + typedef aligned_space SpaceType; + SpaceType my_space; + static const size_t MaxSize = sizeof(SpaceType); + + public: + variant() { (void) new(&my_space) Wrapper(default_constructed()); } + + template + variant( const T& x ) { + do_if::value>::construct(&my_space,x); + } + + variant(const variant& other) { + const WrapperBase * h = punned_cast(&(other.my_space)); + h->CopyTo(&my_space); + } + + // assignment must destroy and re-create the Wrapper type, as there is no way + // to create a Wrapper-to-Wrapper assign even if we find they agree in type. + void operator=( const variant& rhs ) { + if(&rhs != this) { + WrapperBase *h = punned_cast(&my_space); + h->~WrapperBase(); + const WrapperBase *ch = punned_cast(&(rhs.my_space)); + ch->CopyTo(&my_space); + } + } + + template + const U& variant_cast_to() const { + const Wrapper *h = dynamic_cast*>(punned_cast(&my_space)); + if(!h) { + throw_exception(exception_id::bad_tagged_msg_cast); + } + return h->value(); + } + template + bool variant_is_a() const { return dynamic_cast*>(punned_cast(&my_space)) != nullptr; } + + bool variant_is_default_constructed() const {return variant_is_a();} + + ~variant() { + WrapperBase *h = punned_cast(&my_space); + h->~WrapperBase(); + } + }; //class variant + + TagType my_tag; + variant my_msg; +public: + tagged_msg(): my_tag(TagType(~0)), my_msg(){} + + template + tagged_msg(T const &index, R const &value) : my_tag(index), my_msg(value) {} + + template + tagged_msg(T const &index, R (&value)[N]) : my_tag(index), my_msg(value) {} + + void set_tag(TagType const &index) {my_tag = index;} + TagType tag() const {return my_tag;} + + template + const V& cast_to() const {return my_msg.template variant_cast_to();} + + template + bool is_a() const {return my_msg.template variant_is_a();} + + bool is_default_constructed() const {return my_msg.variant_is_default_constructed();} +}; // class tagged_msg + +// template to simplify cast and test for tagged_msg in template contexts +template +const V& cast_to(T const &t) { return t.template cast_to(); } + +template +bool is_a(T const &t) { return t.template is_a(); } + +enum op_stat { WAIT = 0, SUCCEEDED, FAILED }; + +#endif /* __TBB__flow_graph_types_impl_H */ diff --git a/src/3rd party/oneapi/tbb/detail/_hash_compare.h b/src/3rd party/oneapi/tbb/detail/_hash_compare.h new file mode 100644 index 0000000000..5439264f59 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_hash_compare.h @@ -0,0 +1,147 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__hash_compare_H +#define __TBB_detail__hash_compare_H + +#include + +#include "_containers_helpers.h" + +namespace tbb { +namespace detail { +namespace d1 { + +template +class hash_compare { + using is_transparent_hash = has_transparent_key_equal; +public: + using hasher = Hash; + using key_equal = typename is_transparent_hash::type; + + hash_compare() = default; + hash_compare( hasher hash, key_equal equal ) : my_hasher(hash), my_equal(equal) {} + + std::size_t operator()( const Key& key ) const { + return std::size_t(my_hasher(key)); + } + + bool operator()( const Key& key1, const Key& key2 ) const { + return my_equal(key1, key2); + } + + template ::type> + std::size_t operator()( const K& key ) const { + return std::size_t(my_hasher(key)); + } + + template ::type> + bool operator()( const K1& key1, const K2& key2 ) const { + return my_equal(key1, key2); + } + + hasher hash_function() const { + return my_hasher; + } + + key_equal key_eq() const { + return my_equal; + } + + +private: + hasher my_hasher; + key_equal my_equal; +}; // class hash_compare + +//! hash_compare that is default argument for concurrent_hash_map +template +class tbb_hash_compare { +public: + std::size_t hash( const Key& a ) const { return my_hash_func(a); } +#if defined(_MSC_VER) && _MSC_VER <= 1900 +#pragma warning (push) +// MSVC 2015 throws a strange warning: 'std::size_t': forcing value to bool 'true' or 'false' +#pragma warning (disable: 4800) +#endif + bool equal( const Key& a, const Key& b ) const { return my_key_equal(a, b); } +#if defined(_MSC_VER) && _MSC_VER <= 1900 +#pragma warning (pop) +#endif +private: + std::hash my_hash_func; + std::equal_to my_key_equal; +}; + +} // namespace d1 +#if __TBB_CPP20_CONCEPTS_PRESENT +inline namespace d0 { + +template +concept hash_compare = std::copy_constructible && + requires( const std::remove_reference_t& hc, const Key& key1, const Key& key2 ) { + { hc.hash(key1) } -> std::same_as; + { hc.equal(key1, key2) } -> std::convertible_to; + }; + +} // namespace d0 +#endif // __TBB_CPP20_CONCEPTS_PRESENT +} // namespace detail +} // namespace tbb + +#if TBB_DEFINE_STD_HASH_SPECIALIZATIONS + +namespace std { + +template +struct hash> { +public: + std::size_t operator()( const std::pair& p ) const { + return first_hash(p.first) ^ second_hash(p.second); + } + +private: + std::hash first_hash; + std::hash second_hash; +}; // struct hash + +// Apple clang and MSVC defines their own specializations for std::hash> +#if !(_LIBCPP_VERSION) && !(_CPPLIB_VER) + +template +struct hash> { +public: + std::size_t operator()( const std::basic_string& s ) const { + std::size_t h = 0; + for ( const CharT* c = s.c_str(); *c; ++c ) { + h = h * hash_multiplier ^ char_hash(*c); + } + return h; + } + +private: + static constexpr std::size_t hash_multiplier = tbb::detail::select_size_t_constant<2654435769U, 11400714819323198485ULL>::value; + + std::hash char_hash; +}; // struct hash + +#endif // !(_LIBCPP_VERSION || _CPPLIB_VER) + +} // namespace std + +#endif // TBB_DEFINE_STD_HASH_SPECIALIZATIONS + +#endif // __TBB_detail__hash_compare_H diff --git a/src/3rd party/oneapi/tbb/detail/_intrusive_list_node.h b/src/3rd party/oneapi/tbb/detail/_intrusive_list_node.h new file mode 100644 index 0000000000..69286c8fab --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_intrusive_list_node.h @@ -0,0 +1,41 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _TBB_detail__intrusive_list_node_H +#define _TBB_detail__intrusive_list_node_H + +namespace tbb { +namespace detail { +namespace d1 { + +//! Data structure to be inherited by the types that can form intrusive lists. +/** Intrusive list is formed by means of the member_intrusive_list template class. + Note that type T must derive from intrusive_list_node either publicly or + declare instantiation member_intrusive_list as a friend. + This class implements a limited subset of std::list interface. **/ +struct intrusive_list_node { + intrusive_list_node* my_prev_node{}; + intrusive_list_node* my_next_node{}; +#if TBB_USE_ASSERT + intrusive_list_node() { my_prev_node = my_next_node = this; } +#endif /* TBB_USE_ASSERT */ +}; + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif // _TBB_detail__intrusive_list_node_H diff --git a/src/3rd party/oneapi/tbb/detail/_machine.h b/src/3rd party/oneapi/tbb/detail/_machine.h new file mode 100644 index 0000000000..410d3e3920 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_machine.h @@ -0,0 +1,371 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__machine_H +#define __TBB_detail__machine_H + +#include "_config.h" +#include "_assert.h" + +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#ifdef __TBBMALLOC_BUILD +// we only need this function; do not include whole of windows.h +extern "C" __declspec(dllimport) int __stdcall SwitchToThread(void); +#endif +#ifdef _MSC_VER +#if __TBB_x86_64 || __TBB_x86_32 +#pragma intrinsic(__rdtsc) +#endif +#endif +#endif +#if __TBB_x86_64 || __TBB_x86_32 +#include // _mm_pause +#endif +#if (_WIN32) +#include // _control87 +#endif + +#if __TBB_GLIBCXX_THIS_THREAD_YIELD_BROKEN +#include // sched_yield +#else +#include // std::this_thread::yield() +#endif + +namespace tbb { +namespace detail { +inline namespace d0 { + +//-------------------------------------------------------------------------------------------------- +// Yield implementation +//-------------------------------------------------------------------------------------------------- + +#if __TBB_GLIBCXX_THIS_THREAD_YIELD_BROKEN +inline void yield() { + int err = sched_yield(); + __TBB_ASSERT_EX(err == 0, "sched_yield has failed"); +} +#elif __TBBMALLOC_BUILD && _WIN32 +// Use Windows API for yield in tbbmalloc to avoid dependency on C++ runtime with some implementations. +inline void yield() { + SwitchToThread(); +} +#else +using std::this_thread::yield; +#endif + +//-------------------------------------------------------------------------------------------------- +// atomic_fence_seq_cst implementation +//-------------------------------------------------------------------------------------------------- + +inline void atomic_fence_seq_cst() { +#if (__TBB_x86_64 || __TBB_x86_32) && defined(__GNUC__) && __GNUC__ < 11 + unsigned char dummy = 0u; + __asm__ __volatile__ ("lock; notb %0" : "+m" (dummy) :: "memory"); +#else + std::atomic_thread_fence(std::memory_order_seq_cst); +#endif +} + +//-------------------------------------------------------------------------------------------------- +// Pause implementation +//-------------------------------------------------------------------------------------------------- + +inline void machine_pause(int32_t delay) { +#if __TBB_x86_64 || __TBB_x86_32 + while (delay-- > 0) { _mm_pause(); } +#elif __ARM_ARCH_7A__ || __aarch64__ + while (delay-- > 0) { __asm__ __volatile__("isb sy" ::: "memory"); } +#else /* Generic */ + (void)delay; // suppress without including _template_helpers.h + yield(); +#endif +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// tbb::detail::log2() implementation +//////////////////////////////////////////////////////////////////////////////////////////////////// +// TODO: Use log2p1() function that will be available in C++20 standard + +#if defined(__GNUC__) || defined(__clang__) +namespace gnu_builtins { + inline uintptr_t clz(unsigned int x) { return static_cast(__builtin_clz(x)); } + inline uintptr_t clz(unsigned long int x) { return static_cast(__builtin_clzl(x)); } + inline uintptr_t clz(unsigned long long int x) { return static_cast(__builtin_clzll(x)); } +} +#elif defined(_MSC_VER) +#pragma intrinsic(__TBB_W(_BitScanReverse)) +namespace msvc_intrinsics { + inline uintptr_t bit_scan_reverse(uintptr_t i) { + unsigned long j; + __TBB_W(_BitScanReverse)( &j, i ); + return j; + } +} +#endif + +template +constexpr std::uintptr_t number_of_bits() { + return sizeof(T) * CHAR_BIT; +} + +// logarithm is the index of the most significant non-zero bit +inline uintptr_t machine_log2(uintptr_t x) { +#if defined(__GNUC__) || defined(__clang__) + // If P is a power of 2 and x() - 1) ^ gnu_builtins::clz(x); +#elif defined(_MSC_VER) + return msvc_intrinsics::bit_scan_reverse(x); +#elif __i386__ || __i386 /*for Sun OS*/ || __MINGW32__ + uintptr_t j, i = x; + __asm__("bsr %1,%0" : "=r"(j) : "r"(i)); + return j; +#elif __powerpc__ || __POWERPC__ + #if __TBB_WORDSIZE==8 + __asm__ __volatile__ ("cntlzd %0,%0" : "+r"(x)); + return 63 - static_cast(x); + #else + __asm__ __volatile__ ("cntlzw %0,%0" : "+r"(x)); + return 31 - static_cast(x); + #endif /*__TBB_WORDSIZE*/ +#elif __sparc + uint64_t count; + // one hot encode + x |= (x >> 1); + x |= (x >> 2); + x |= (x >> 4); + x |= (x >> 8); + x |= (x >> 16); + x |= (x >> 32); + // count 1's + __asm__ ("popc %1, %0" : "=r"(count) : "r"(x) ); + return count - 1; +#else + intptr_t result = 0; + + if( sizeof(x) > 4 && (uintptr_t tmp = x >> 32) ) { x = tmp; result += 32; } + if( uintptr_t tmp = x >> 16 ) { x = tmp; result += 16; } + if( uintptr_t tmp = x >> 8 ) { x = tmp; result += 8; } + if( uintptr_t tmp = x >> 4 ) { x = tmp; result += 4; } + if( uintptr_t tmp = x >> 2 ) { x = tmp; result += 2; } + + return (x & 2) ? result + 1 : result; +#endif +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// tbb::detail::reverse_bits() implementation +//////////////////////////////////////////////////////////////////////////////////////////////////// +#if TBB_USE_CLANG_BITREVERSE_BUILTINS +namespace llvm_builtins { + inline uint8_t builtin_bitreverse(uint8_t x) { return __builtin_bitreverse8 (x); } + inline uint16_t builtin_bitreverse(uint16_t x) { return __builtin_bitreverse16(x); } + inline uint32_t builtin_bitreverse(uint32_t x) { return __builtin_bitreverse32(x); } + inline uint64_t builtin_bitreverse(uint64_t x) { return __builtin_bitreverse64(x); } +} +#else // generic +template +struct reverse { + static const T byte_table[256]; +}; + +template +const T reverse::byte_table[256] = { + 0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0, 0x10, 0x90, 0x50, 0xD0, 0x30, 0xB0, 0x70, 0xF0, + 0x08, 0x88, 0x48, 0xC8, 0x28, 0xA8, 0x68, 0xE8, 0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, 0x78, 0xF8, + 0x04, 0x84, 0x44, 0xC4, 0x24, 0xA4, 0x64, 0xE4, 0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4, + 0x0C, 0x8C, 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC, 0x1C, 0x9C, 0x5C, 0xDC, 0x3C, 0xBC, 0x7C, 0xFC, + 0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, 0x62, 0xE2, 0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2, + 0x0A, 0x8A, 0x4A, 0xCA, 0x2A, 0xAA, 0x6A, 0xEA, 0x1A, 0x9A, 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA, + 0x06, 0x86, 0x46, 0xC6, 0x26, 0xA6, 0x66, 0xE6, 0x16, 0x96, 0x56, 0xD6, 0x36, 0xB6, 0x76, 0xF6, + 0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE, 0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE, + 0x01, 0x81, 0x41, 0xC1, 0x21, 0xA1, 0x61, 0xE1, 0x11, 0x91, 0x51, 0xD1, 0x31, 0xB1, 0x71, 0xF1, + 0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, 0x69, 0xE9, 0x19, 0x99, 0x59, 0xD9, 0x39, 0xB9, 0x79, 0xF9, + 0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5, 0x15, 0x95, 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5, + 0x0D, 0x8D, 0x4D, 0xCD, 0x2D, 0xAD, 0x6D, 0xED, 0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, 0x7D, 0xFD, + 0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3, 0x13, 0x93, 0x53, 0xD3, 0x33, 0xB3, 0x73, 0xF3, + 0x0B, 0x8B, 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB, 0x1B, 0x9B, 0x5B, 0xDB, 0x3B, 0xBB, 0x7B, 0xFB, + 0x07, 0x87, 0x47, 0xC7, 0x27, 0xA7, 0x67, 0xE7, 0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7, + 0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF, 0x1F, 0x9F, 0x5F, 0xDF, 0x3F, 0xBF, 0x7F, 0xFF +}; + +inline unsigned char reverse_byte(unsigned char src) { + return reverse::byte_table[src]; +} +#endif // TBB_USE_CLANG_BITREVERSE_BUILTINS + +template +T machine_reverse_bits(T src) { +#if TBB_USE_CLANG_BITREVERSE_BUILTINS + return builtin_bitreverse(fixed_width_cast(src)); +#else /* Generic */ + T dst; + unsigned char *original = reinterpret_cast(&src); + unsigned char *reversed = reinterpret_cast(&dst); + + for ( int i = sizeof(T) - 1; i >= 0; i-- ) { + reversed[i] = reverse_byte( original[sizeof(T) - i - 1] ); + } + + return dst; +#endif // TBB_USE_CLANG_BITREVERSE_BUILTINS +} + +} // inline namespace d0 + +namespace d1 { + +#if (_WIN32) +// API to retrieve/update FPU control setting +#define __TBB_CPU_CTL_ENV_PRESENT 1 +struct cpu_ctl_env { + unsigned int x87cw{}; +#if (__TBB_x86_64) + // Changing the infinity mode or the floating-point precision is not supported on x64. + // The attempt causes an assertion. See + // https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/control87-controlfp-control87-2 + static constexpr unsigned int X87CW_CONTROL_MASK = _MCW_DN | _MCW_EM | _MCW_RC; +#else + static constexpr unsigned int X87CW_CONTROL_MASK = ~0U; +#endif +#if (__TBB_x86_32 || __TBB_x86_64) + unsigned int mxcsr{}; + static constexpr unsigned int MXCSR_CONTROL_MASK = ~0x3fu; /* all except last six status bits */ +#endif + + bool operator!=( const cpu_ctl_env& ctl ) const { + return +#if (__TBB_x86_32 || __TBB_x86_64) + mxcsr != ctl.mxcsr || +#endif + x87cw != ctl.x87cw; + } + void get_env() { + x87cw = _control87(0, 0); +#if (__TBB_x86_32 || __TBB_x86_64) + mxcsr = _mm_getcsr(); +#endif + } + void set_env() const { + _control87(x87cw, X87CW_CONTROL_MASK); +#if (__TBB_x86_32 || __TBB_x86_64) + _mm_setcsr(mxcsr & MXCSR_CONTROL_MASK); +#endif + } +}; +#elif (__TBB_x86_32 || __TBB_x86_64) +// API to retrieve/update FPU control setting +#define __TBB_CPU_CTL_ENV_PRESENT 1 +struct cpu_ctl_env { + int mxcsr{}; + short x87cw{}; + static const int MXCSR_CONTROL_MASK = ~0x3f; /* all except last six status bits */ + + bool operator!=(const cpu_ctl_env& ctl) const { + return mxcsr != ctl.mxcsr || x87cw != ctl.x87cw; + } + void get_env() { + __asm__ __volatile__( + "stmxcsr %0\n\t" + "fstcw %1" + : "=m"(mxcsr), "=m"(x87cw) + ); + mxcsr &= MXCSR_CONTROL_MASK; + } + void set_env() const { + __asm__ __volatile__( + "ldmxcsr %0\n\t" + "fldcw %1" + : : "m"(mxcsr), "m"(x87cw) + ); + } +}; +#endif + +} // namespace d1 + +} // namespace detail +} // namespace tbb + +#if !__TBB_CPU_CTL_ENV_PRESENT +#include + +#include + +namespace tbb { +namespace detail { + +namespace r1 { +void* __TBB_EXPORTED_FUNC cache_aligned_allocate(std::size_t size); +void __TBB_EXPORTED_FUNC cache_aligned_deallocate(void* p); +} // namespace r1 + +namespace d1 { + +class cpu_ctl_env { + fenv_t *my_fenv_ptr; +public: + cpu_ctl_env() : my_fenv_ptr(nullptr) {} + ~cpu_ctl_env() { + if ( my_fenv_ptr ) + r1::cache_aligned_deallocate( (void*)my_fenv_ptr ); + } + // It is possible not to copy memory but just to copy pointers but the following issues should be addressed: + // 1. The arena lifetime and the context lifetime are independent; + // 2. The user is allowed to recapture different FPU settings to context so 'current FPU settings' inside + // dispatch loop may become invalid. + // But do we really want to improve the fenv implementation? It seems to be better to replace the fenv implementation + // with a platform specific implementation. + cpu_ctl_env( const cpu_ctl_env &src ) : my_fenv_ptr(nullptr) { + *this = src; + } + cpu_ctl_env& operator=( const cpu_ctl_env &src ) { + __TBB_ASSERT( src.my_fenv_ptr, nullptr); + if ( !my_fenv_ptr ) + my_fenv_ptr = (fenv_t*)r1::cache_aligned_allocate(sizeof(fenv_t)); + *my_fenv_ptr = *src.my_fenv_ptr; + return *this; + } + bool operator!=( const cpu_ctl_env &ctl ) const { + __TBB_ASSERT( my_fenv_ptr, "cpu_ctl_env is not initialized." ); + __TBB_ASSERT( ctl.my_fenv_ptr, "cpu_ctl_env is not initialized." ); + return std::memcmp( (void*)my_fenv_ptr, (void*)ctl.my_fenv_ptr, sizeof(fenv_t) ); + } + void get_env () { + if ( !my_fenv_ptr ) + my_fenv_ptr = (fenv_t*)r1::cache_aligned_allocate(sizeof(fenv_t)); + fegetenv( my_fenv_ptr ); + } + const cpu_ctl_env& set_env () const { + __TBB_ASSERT( my_fenv_ptr, "cpu_ctl_env is not initialized." ); + fesetenv( my_fenv_ptr ); + return *this; + } +}; + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif /* !__TBB_CPU_CTL_ENV_PRESENT */ + +#endif // __TBB_detail__machine_H diff --git a/src/3rd party/oneapi/tbb/detail/_mutex_common.h b/src/3rd party/oneapi/tbb/detail/_mutex_common.h new file mode 100644 index 0000000000..4650c19268 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_mutex_common.h @@ -0,0 +1,61 @@ +/* + Copyright (c) 2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__mutex_common_H +#define __TBB_detail__mutex_common_H + +#include "_config.h" +#include "_utils.h" + +#if __TBB_CPP20_CONCEPTS_PRESENT +#include + +namespace tbb { +namespace detail { +inline namespace d0 { + +template +concept mutex_scoped_lock = std::default_initializable && + std::constructible_from && + requires( Lock& lock, Mutex& mutex ) { + lock.acquire(mutex); + { lock.try_acquire(mutex) } -> adaptive_same_as; + lock.release(); + }; + +template +concept rw_mutex_scoped_lock = mutex_scoped_lock && + std::constructible_from && + requires( Lock& lock, Mutex& mutex ) { + lock.acquire(mutex, false); + { lock.try_acquire(mutex, false) } -> adaptive_same_as; + { lock.upgrade_to_writer() } -> adaptive_same_as; + { lock.downgrade_to_reader() } -> adaptive_same_as; + }; + +template +concept scoped_lockable = mutex_scoped_lock; + +template +concept rw_scoped_lockable = scoped_lockable && + rw_mutex_scoped_lock; + +} // namespace d0 +} // namespace detail +} // namespace tbb + +#endif // __TBB_CPP20_CONCEPTS_PRESENT +#endif // __TBB_detail__mutex_common_H diff --git a/src/3rd party/tbb/internal/_warning_suppress_enable_notice.h b/src/3rd party/oneapi/tbb/detail/_namespace_injection.h similarity index 54% rename from src/3rd party/tbb/internal/_warning_suppress_enable_notice.h rename to src/3rd party/oneapi/tbb/detail/_namespace_injection.h index cd2bc8a582..2e1df30931 100644 --- a/src/3rd party/tbb/internal/_warning_suppress_enable_notice.h +++ b/src/3rd party/oneapi/tbb/detail/_namespace_injection.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2020-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,19 +14,11 @@ limitations under the License. */ -#include "../tbb_config.h" +// All public entities of the OneAPI Spec are available under oneapi namespace -#if __TBB_SUPPRESS_INTERNAL_DEPRECATED_MESSAGES +// Define tbb namespace first as it might not be known yet +namespace tbb {} -#if _MSC_VER || __INTEL_COMPILER -#pragma warning( push ) -#pragma warning( disable: 4996 ) -#elif __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#elif __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wdeprecated-declarations" -#endif - -#endif // __TBB_SUPPRESS_INTERNAL_DEPRECATED_MESSAGES +namespace oneapi { +namespace tbb = ::tbb; +} diff --git a/src/3rd party/oneapi/tbb/detail/_node_handle.h b/src/3rd party/oneapi/tbb/detail/_node_handle.h new file mode 100644 index 0000000000..265be07555 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_node_handle.h @@ -0,0 +1,162 @@ +/* + Copyright (c) 2019-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__node_handle_H +#define __TBB_detail__node_handle_H + +#include "_allocator_traits.h" +#include "_assert.h" + +namespace tbb { +namespace detail { +namespace d1 { + +// A structure to access private node handle methods in internal TBB classes +// Regular friend declaration is not convenient because classes which use node handle +// can be placed in the different versioning namespaces. +struct node_handle_accessor { + template + static typename NodeHandleType::node* get_node_ptr( NodeHandleType& nh ) { + return nh.get_node_ptr(); + } + + template + static NodeHandleType construct( typename NodeHandleType::node* node_ptr ) { + return NodeHandleType{node_ptr}; + } + + template + static void deactivate( NodeHandleType& nh ) { + nh.deactivate(); + } +}; // struct node_handle_accessor + +template +class node_handle_base { +public: + using allocator_type = Allocator; +protected: + using node = Node; + using allocator_traits_type = tbb::detail::allocator_traits; +public: + + node_handle_base() : my_node(nullptr), my_allocator() {} + node_handle_base(node_handle_base&& nh) : my_node(nh.my_node), + my_allocator(std::move(nh.my_allocator)) { + nh.my_node = nullptr; + } + + __TBB_nodiscard bool empty() const { return my_node == nullptr; } + explicit operator bool() const { return my_node != nullptr; } + + ~node_handle_base() { internal_destroy(); } + + node_handle_base& operator=( node_handle_base&& nh ) { + internal_destroy(); + my_node = nh.my_node; + move_assign_allocators(my_allocator, nh.my_allocator); + nh.deactivate(); + return *this; + } + + void swap( node_handle_base& nh ) { + using std::swap; + swap(my_node, nh.my_node); + swap_allocators(my_allocator, nh.my_allocator); + } + + allocator_type get_allocator() const { + return my_allocator; + } + +protected: + node_handle_base( node* n ) : my_node(n) {} + + void internal_destroy() { + if(my_node != nullptr) { + allocator_traits_type::destroy(my_allocator, my_node->storage()); + typename allocator_traits_type::template rebind_alloc node_allocator(my_allocator); + node_allocator.deallocate(my_node, 1); + } + } + + node* get_node_ptr() { return my_node; } + + void deactivate() { my_node = nullptr; } + + node* my_node; + allocator_type my_allocator; +}; + +// node handle for maps +template +class node_handle : public node_handle_base { + using base_type = node_handle_base; +public: + using key_type = Key; + using mapped_type = typename Value::second_type; + using allocator_type = typename base_type::allocator_type; + + node_handle() = default; + + key_type& key() const { + __TBB_ASSERT(!this->empty(), "Cannot get key from the empty node_type object"); + return *const_cast(&(this->my_node->value().first)); + } + + mapped_type& mapped() const { + __TBB_ASSERT(!this->empty(), "Cannot get mapped value from the empty node_type object"); + return this->my_node->value().second; + } + +private: + friend struct node_handle_accessor; + + node_handle( typename base_type::node* n ) : base_type(n) {} +}; // class node_handle + +// node handle for sets +template +class node_handle : public node_handle_base { + using base_type = node_handle_base; +public: + using value_type = Key; + using allocator_type = typename base_type::allocator_type; + + node_handle() = default; + + value_type& value() const { + __TBB_ASSERT(!this->empty(), "Cannot get value from the empty node_type object"); + return *const_cast(&(this->my_node->value())); + } + +private: + friend struct node_handle_accessor; + + node_handle( typename base_type::node* n ) : base_type(n) {} +}; // class node_handle + +template +void swap( node_handle& lhs, + node_handle& rhs ) { + return lhs.swap(rhs); +} + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__node_handle_H diff --git a/src/3rd party/oneapi/tbb/detail/_pipeline_filters.h b/src/3rd party/oneapi/tbb/detail/_pipeline_filters.h new file mode 100644 index 0000000000..8121946729 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_pipeline_filters.h @@ -0,0 +1,461 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_parallel_filters_H +#define __TBB_parallel_filters_H + +#include "_config.h" +#include "_task.h" +#include "_pipeline_filters_deduction.h" +#include "../tbb_allocator.h" + +#include +#include + +namespace tbb { +namespace detail { + +namespace d1 { +class base_filter; +} + +namespace d2 { +template +__TBB_requires(std::copyable) +class input_node; +} + +namespace r1 { +TBB_EXPORT void __TBB_EXPORTED_FUNC set_end_of_input(d1::base_filter&); +class pipeline; +class stage_task; +class input_buffer; +} + +namespace d1 { +class filter_node; + +//! A stage in a pipeline. +/** @ingroup algorithms */ +class base_filter{ +private: + //! Value used to mark "not in pipeline" + static base_filter* not_in_pipeline() { return reinterpret_cast(std::intptr_t(-1)); } +public: + //! The lowest bit 0 is for parallel vs serial + static constexpr unsigned int filter_is_serial = 0x1; + + //! 2nd bit distinguishes ordered vs unordered filters. + static constexpr unsigned int filter_is_out_of_order = 0x1<<1; + + //! 3rd bit marks input filters emitting small objects + static constexpr unsigned int filter_may_emit_null = 0x1<<2; + + base_filter(const base_filter&) = delete; + base_filter& operator=(const base_filter&) = delete; + +protected: + explicit base_filter( unsigned int m ) : + next_filter_in_pipeline(not_in_pipeline()), + my_input_buffer(nullptr), + my_filter_mode(m), + my_pipeline(nullptr) + {} + + // signal end-of-input for concrete_filters + void set_end_of_input() { + r1::set_end_of_input(*this); + } + +public: + //! True if filter is serial. + bool is_serial() const { + return bool( my_filter_mode & filter_is_serial ); + } + + //! True if filter must receive stream in order. + bool is_ordered() const { + return (my_filter_mode & filter_is_serial) && !(my_filter_mode & filter_is_out_of_order); + } + + //! true if an input filter can emit null + bool object_may_be_null() { + return ( my_filter_mode & filter_may_emit_null ) == filter_may_emit_null; + } + + //! Operate on an item from the input stream, and return item for output stream. + /** Returns nullptr if filter is a sink. */ + virtual void* operator()( void* item ) = 0; + + //! Destroy filter. + virtual ~base_filter() {}; + + //! Destroys item if pipeline was cancelled. + /** Required to prevent memory leaks. + Note it can be called concurrently even for serial filters.*/ + virtual void finalize( void* /*item*/ ) {} + +private: + //! Pointer to next filter in the pipeline. + base_filter* next_filter_in_pipeline; + + //! Buffer for incoming tokens, or nullptr if not required. + /** The buffer is required if the filter is serial. */ + r1::input_buffer* my_input_buffer; + + friend class r1::stage_task; + friend class r1::pipeline; + friend void r1::set_end_of_input(d1::base_filter&); + + //! Storage for filter mode and dynamically checked implementation version. + const unsigned int my_filter_mode; + + //! Pointer to the pipeline. + r1::pipeline* my_pipeline; +}; + +template +class concrete_filter; + +//! input_filter control to signal end-of-input for parallel_pipeline +class flow_control { + bool is_pipeline_stopped = false; + flow_control() = default; + template friend class concrete_filter; + template + __TBB_requires(std::copyable) + friend class d2::input_node; +public: + void stop() { is_pipeline_stopped = true; } +}; + +// Emulate std::is_trivially_copyable (false positives not allowed, false negatives suboptimal but safe). +#if __TBB_CPP11_TYPE_PROPERTIES_PRESENT +template using tbb_trivially_copyable = std::is_trivially_copyable; +#else +template struct tbb_trivially_copyable { enum { value = false }; }; +template struct tbb_trivially_copyable < T* > { enum { value = true }; }; +template<> struct tbb_trivially_copyable < bool > { enum { value = true }; }; +template<> struct tbb_trivially_copyable < char > { enum { value = true }; }; +template<> struct tbb_trivially_copyable < signed char > { enum { value = true }; }; +template<> struct tbb_trivially_copyable { enum { value = true }; }; +template<> struct tbb_trivially_copyable < short > { enum { value = true }; }; +template<> struct tbb_trivially_copyable { enum { value = true }; }; +template<> struct tbb_trivially_copyable < int > { enum { value = true }; }; +template<> struct tbb_trivially_copyable { enum { value = true }; }; +template<> struct tbb_trivially_copyable < long > { enum { value = true }; }; +template<> struct tbb_trivially_copyable { enum { value = true }; }; +template<> struct tbb_trivially_copyable < long long> { enum { value = true }; }; +template<> struct tbb_trivially_copyable { enum { value = true }; }; +template<> struct tbb_trivially_copyable < float > { enum { value = true }; }; +template<> struct tbb_trivially_copyable < double > { enum { value = true }; }; +template<> struct tbb_trivially_copyable < long double > { enum { value = true }; }; +#endif // __TBB_CPP11_TYPE_PROPERTIES_PRESENT + +template +struct use_allocator { + static constexpr bool value = sizeof(T) > sizeof(void *) || !tbb_trivially_copyable::value; +}; + +// A helper class to customize how a type is passed between filters. +// Usage: token_helper::value> +template struct token_helper; + +// using tbb_allocator +template +struct token_helper { + using pointer = T*; + using value_type = T; + static pointer create_token(value_type && source) { + return new (r1::allocate_memory(sizeof(T))) T(std::move(source)); + } + static value_type & token(pointer & t) { return *t; } + static void * cast_to_void_ptr(pointer ref) { return reinterpret_cast(ref); } + static pointer cast_from_void_ptr(void * ref) { return reinterpret_cast(ref); } + static void destroy_token(pointer token) { + token->~value_type(); + r1::deallocate_memory(token); + } +}; + +// pointer specialization +template +struct token_helper { + using pointer = T*; + using value_type = T*; + static pointer create_token(const value_type & source) { return source; } + static value_type & token(pointer & t) { return t; } + static void * cast_to_void_ptr(pointer ref) { return reinterpret_cast(ref); } + static pointer cast_from_void_ptr(void * ref) { return reinterpret_cast(ref); } + static void destroy_token( pointer /*token*/) {} +}; + +// converting type to and from void*, passing objects directly +template +struct token_helper { + typedef union { + T actual_value; + void * void_overlay; + } type_to_void_ptr_map; + using pointer = T; // not really a pointer in this case. + using value_type = T; + static pointer create_token(const value_type & source) { return source; } + static value_type & token(pointer & t) { return t; } + static void * cast_to_void_ptr(pointer ref) { + type_to_void_ptr_map mymap; + mymap.void_overlay = nullptr; + mymap.actual_value = ref; + return mymap.void_overlay; + } + static pointer cast_from_void_ptr(void * ref) { + type_to_void_ptr_map mymap; + mymap.void_overlay = ref; + return mymap.actual_value; + } + static void destroy_token( pointer /*token*/) {} +}; + +// intermediate +template +class concrete_filter: public base_filter { + const Body& my_body; + using input_helper = token_helper::value>; + using input_pointer = typename input_helper::pointer; + using output_helper = token_helper::value>; + using output_pointer = typename output_helper::pointer; + + void* operator()(void* input) override { + input_pointer temp_input = input_helper::cast_from_void_ptr(input); + output_pointer temp_output = output_helper::create_token(tbb::detail::invoke(my_body, std::move(input_helper::token(temp_input)))); + input_helper::destroy_token(temp_input); + return output_helper::cast_to_void_ptr(temp_output); + } + + void finalize(void * input) override { + input_pointer temp_input = input_helper::cast_from_void_ptr(input); + input_helper::destroy_token(temp_input); + } + +public: + concrete_filter(unsigned int m, const Body& body) : base_filter(m), my_body(body) {} +}; + +// input +template +class concrete_filter: public base_filter { + const Body& my_body; + using output_helper = token_helper::value>; + using output_pointer = typename output_helper::pointer; + + void* operator()(void*) override { + flow_control control; + output_pointer temp_output = output_helper::create_token(my_body(control)); + if(control.is_pipeline_stopped) { + output_helper::destroy_token(temp_output); + set_end_of_input(); + return nullptr; + } + return output_helper::cast_to_void_ptr(temp_output); + } + +public: + concrete_filter(unsigned int m, const Body& body) : + base_filter(m | filter_may_emit_null), + my_body(body) + {} +}; + +// output +template +class concrete_filter: public base_filter { + const Body& my_body; + using input_helper = token_helper::value>; + using input_pointer = typename input_helper::pointer; + + void* operator()(void* input) override { + input_pointer temp_input = input_helper::cast_from_void_ptr(input); + tbb::detail::invoke(my_body, std::move(input_helper::token(temp_input))); + input_helper::destroy_token(temp_input); + return nullptr; + } + void finalize(void* input) override { + input_pointer temp_input = input_helper::cast_from_void_ptr(input); + input_helper::destroy_token(temp_input); + } + +public: + concrete_filter(unsigned int m, const Body& body) : base_filter(m), my_body(body) {} +}; + +template +class concrete_filter: public base_filter { + const Body& my_body; + + void* operator()(void*) override { + flow_control control; + my_body(control); + void* output = control.is_pipeline_stopped ? nullptr : (void*)(std::intptr_t)-1; + return output; + } +public: + concrete_filter(unsigned int m, const Body& body) : base_filter(m), my_body(body) {} +}; + +class filter_node_ptr { + filter_node * my_node; + +public: + filter_node_ptr() : my_node(nullptr) {} + filter_node_ptr(filter_node *); + ~filter_node_ptr(); + filter_node_ptr(const filter_node_ptr &); + filter_node_ptr(filter_node_ptr &&); + void operator=(filter_node *); + void operator=(const filter_node_ptr &); + void operator=(filter_node_ptr &&); + filter_node& operator*() const; + operator bool() const; +}; + +//! Abstract base class that represents a node in a parse tree underlying a filter class. +/** These nodes are always heap-allocated and can be shared by filter objects. */ +class filter_node { + /** Count must be atomic because it is hidden state for user, but might be shared by threads. */ + std::atomic ref_count; +public: + filter_node_ptr left; + filter_node_ptr right; +protected: + filter_node() : ref_count(0), left(nullptr), right(nullptr) { +#ifdef __TBB_TEST_FILTER_NODE_COUNT + ++(__TBB_TEST_FILTER_NODE_COUNT); +#endif + } +public: + filter_node(const filter_node_ptr& x, const filter_node_ptr& y) : filter_node(){ + left = x; + right = y; + } + filter_node(const filter_node&) = delete; + filter_node& operator=(const filter_node&) = delete; + + //! Add concrete_filter to pipeline + virtual base_filter* create_filter() const { + __TBB_ASSERT(false, "method of non-leaf was called"); + return nullptr; + } + + //! Increment reference count + void add_ref() { ref_count.fetch_add(1, std::memory_order_relaxed); } + + //! Decrement reference count and delete if it becomes zero. + void remove_ref() { + __TBB_ASSERT(ref_count>0,"ref_count underflow"); + if( ref_count.fetch_sub(1, std::memory_order_relaxed) == 1 ) { + this->~filter_node(); + r1::deallocate_memory(this); + } + } + + virtual ~filter_node() { +#ifdef __TBB_TEST_FILTER_NODE_COUNT + --(__TBB_TEST_FILTER_NODE_COUNT); +#endif + } +}; + +inline filter_node_ptr::filter_node_ptr(filter_node * nd) : my_node(nd) { + if (my_node) { + my_node->add_ref(); + } +} + +inline filter_node_ptr::~filter_node_ptr() { + if (my_node) { + my_node->remove_ref(); + } +} + +inline filter_node_ptr::filter_node_ptr(const filter_node_ptr & rhs) : my_node(rhs.my_node) { + if (my_node) { + my_node->add_ref(); + } +} + +inline filter_node_ptr::filter_node_ptr(filter_node_ptr && rhs) : my_node(rhs.my_node) { + rhs.my_node = nullptr; +} + +inline void filter_node_ptr::operator=(filter_node * rhs) { + // Order of operations below carefully chosen so that reference counts remain correct + // in unlikely event that remove_ref throws exception. + filter_node* old = my_node; + my_node = rhs; + if (my_node) { + my_node->add_ref(); + } + if (old) { + old->remove_ref(); + } +} + +inline void filter_node_ptr::operator=(const filter_node_ptr & rhs) { + *this = rhs.my_node; +} + +inline void filter_node_ptr::operator=(filter_node_ptr && rhs) { + filter_node* old = my_node; + my_node = rhs.my_node; + rhs.my_node = nullptr; + if (old) { + old->remove_ref(); + } +} + +inline filter_node& filter_node_ptr::operator*() const{ + __TBB_ASSERT(my_node,"nullptr node is used"); + return *my_node; +} + +inline filter_node_ptr::operator bool() const { + return my_node != nullptr; +} + +//! Node in parse tree representing result of make_filter. +template +class filter_node_leaf: public filter_node { + const unsigned int my_mode; + const Body my_body; + base_filter* create_filter() const override { + return new(r1::allocate_memory(sizeof(concrete_filter))) concrete_filter(my_mode,my_body); + } +public: + filter_node_leaf( unsigned int m, const Body& b ) : my_mode(m), my_body(b) {} +}; + + +template ::input_type> +using filter_input = typename std::conditional::value, void, Input>::type; + +template +using filter_output = typename filter_body_types::output_type; + +} // namespace d1 +} // namespace detail +} // namespace tbb + + +#endif /* __TBB_parallel_filters_H */ diff --git a/src/3rd party/oneapi/tbb/detail/_pipeline_filters_deduction.h b/src/3rd party/oneapi/tbb/detail/_pipeline_filters_deduction.h new file mode 100644 index 0000000000..c1a6e8aed7 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_pipeline_filters_deduction.h @@ -0,0 +1,46 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__pipeline_filters_deduction_H +#define __TBB__pipeline_filters_deduction_H + +#include "_config.h" +#include +#include + +namespace tbb { +namespace detail { +namespace d1 { + +template +struct declare_filter_types { + using input_type = typename std::remove_const::type>::type; + using output_type = typename std::remove_const::type>::type; +}; + +template struct filter_body_types; + +template +struct filter_body_types : declare_filter_types {}; + +template +struct filter_body_types : declare_filter_types {}; + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif // __TBB__pipeline_filters_deduction_H diff --git a/src/3rd party/oneapi/tbb/detail/_range_common.h b/src/3rd party/oneapi/tbb/detail/_range_common.h new file mode 100644 index 0000000000..ac219587d7 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_range_common.h @@ -0,0 +1,133 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__range_common_H +#define __TBB_detail__range_common_H + +#include "_config.h" +#include "_utils.h" +#if __TBB_CPP20_CONCEPTS_PRESENT +#include +#endif +#include + +namespace tbb { +namespace detail { +inline namespace d0 { + +//! Dummy type that distinguishes splitting constructor from copy constructor. +/** + * See description of parallel_for and parallel_reduce for example usages. + * @ingroup algorithms + */ +class split {}; + +//! Type enables transmission of splitting proportion from partitioners to range objects +/** + * In order to make use of such facility Range objects must implement + * splitting constructor with this type passed. + */ +class proportional_split : no_assign { +public: + proportional_split(size_t _left = 1, size_t _right = 1) : my_left(_left), my_right(_right) { } + + size_t left() const { return my_left; } + size_t right() const { return my_right; } + + // used when range does not support proportional split + explicit operator split() const { return split(); } + +private: + size_t my_left, my_right; +}; + +template +struct range_split_object_provider { + template + static split get( PartitionerSplitType& ) { return split(); } +}; + +template +struct range_split_object_provider::value>::type> { + template + static PartitionerSplitType& get( PartitionerSplitType& split_obj ) { return split_obj; } +}; + +template +auto get_range_split_object( PartitionerSplitType& split_obj ) +-> decltype(range_split_object_provider::get(split_obj)) { + return range_split_object_provider::get(split_obj); +} + +template +using range_iterator_type = decltype(std::begin(std::declval())); + +#if __TBB_CPP20_CONCEPTS_PRESENT +template +using iterator_reference_type = typename std::iterator_traits::reference; + +template +using range_reference_type = iterator_reference_type>; + +template +concept blocked_range_value = std::copyable && + requires( const std::remove_reference_t& lhs, const std::remove_reference_t& rhs ) { + { lhs < rhs } -> relaxed_convertible_to; + { lhs - rhs } -> std::convertible_to; + { lhs + (rhs - lhs) } -> std::convertible_to; + }; + +template +concept splittable = std::constructible_from; + +template +concept tbb_range = std::copy_constructible && + splittable && + requires( const std::remove_reference_t& range ) { + { range.empty() } -> relaxed_convertible_to; + { range.is_divisible() } -> relaxed_convertible_to; + }; + +template +struct iterator_concept_helper; + +// New specializations should be added in case of using container_based_sequence with +// the new iterator tag types +template +struct iterator_concept_helper { + static constexpr bool value = std::input_iterator; +}; + +template +struct iterator_concept_helper { + static constexpr bool value = std::random_access_iterator; +}; + +template +concept iterator_satisfies = iterator_concept_helper::value; + +template +concept container_based_sequence = requires( Sequence& seq ) { + { std::begin(seq) } -> iterator_satisfies; + { std::end(seq) } -> iterator_satisfies; +}; +#endif // __TBB_CPP20_CONCEPTS_PRESENT +} // namespace d0 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__range_common_H diff --git a/src/3rd party/oneapi/tbb/detail/_rtm_mutex.h b/src/3rd party/oneapi/tbb/detail/_rtm_mutex.h new file mode 100644 index 0000000000..11583bad3a --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_rtm_mutex.h @@ -0,0 +1,162 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__rtm_mutex_impl_H +#define __TBB__rtm_mutex_impl_H + +#include "_assert.h" +#include "_utils.h" +#include "../spin_mutex.h" + +#include "../profiling.h" + +namespace tbb { +namespace detail { +namespace r1 { +struct rtm_mutex_impl; +} +namespace d1 { + +#if _MSC_VER && !defined(__INTEL_COMPILER) + // Suppress warning: structure was padded due to alignment specifier + #pragma warning (push) + #pragma warning (disable: 4324) +#endif + +/** A rtm_mutex is an speculation-enabled spin mutex. + It should be used for locking short critical sections where the lock is + contended but the data it protects are not. If zero-initialized, the + mutex is considered unheld. + @ingroup synchronization */ +class alignas(max_nfs_size) rtm_mutex : private spin_mutex { +private: + enum class rtm_state { + rtm_none, + rtm_transacting, + rtm_real + }; +public: + //! Constructors + rtm_mutex() noexcept { + create_itt_sync(this, "tbb::speculative_spin_mutex", ""); + } + + //! Destructor + ~rtm_mutex() = default; + + //! Represents acquisition of a mutex. + class scoped_lock { + public: + friend class rtm_mutex; + //! Construct lock that has not acquired a mutex. + /** Equivalent to zero-initialization of *this. */ + constexpr scoped_lock() : m_mutex(nullptr), m_transaction_state(rtm_state::rtm_none) {} + + //! Acquire lock on given mutex. + scoped_lock(rtm_mutex& m) : m_mutex(nullptr), m_transaction_state(rtm_state::rtm_none) { + acquire(m); + } + + //! Release lock (if lock is held). + ~scoped_lock() { + if(m_transaction_state != rtm_state::rtm_none) { + release(); + } + } + + //! No Copy + scoped_lock(const scoped_lock&) = delete; + scoped_lock& operator=(const scoped_lock&) = delete; + + //! Acquire lock on given mutex. + void acquire(rtm_mutex& m); + + //! Try acquire lock on given mutex. + bool try_acquire(rtm_mutex& m); + + //! Release lock + void release(); + + private: + rtm_mutex* m_mutex; + rtm_state m_transaction_state; + friend r1::rtm_mutex_impl; + }; + + //! Mutex traits + static constexpr bool is_rw_mutex = false; + static constexpr bool is_recursive_mutex = false; + static constexpr bool is_fair_mutex = false; +private: + friend r1::rtm_mutex_impl; +}; // end of rtm_mutex +} // namespace d1 + +namespace r1 { + //! Internal acquire lock. + // only_speculate == true if we're doing a try_lock, else false. + TBB_EXPORT void __TBB_EXPORTED_FUNC acquire(d1::rtm_mutex&, d1::rtm_mutex::scoped_lock&, bool only_speculate = false); + //! Internal try_acquire lock. + TBB_EXPORT bool __TBB_EXPORTED_FUNC try_acquire(d1::rtm_mutex&, d1::rtm_mutex::scoped_lock&); + //! Internal release lock. + TBB_EXPORT void __TBB_EXPORTED_FUNC release(d1::rtm_mutex::scoped_lock&); +} // namespace r1 + +namespace d1 { +//! Acquire lock on given mutex. +inline void rtm_mutex::scoped_lock::acquire(rtm_mutex& m) { + __TBB_ASSERT(!m_mutex, "lock is already acquired"); + r1::acquire(m, *this); +} + +//! Try acquire lock on given mutex. +inline bool rtm_mutex::scoped_lock::try_acquire(rtm_mutex& m) { + __TBB_ASSERT(!m_mutex, "lock is already acquired"); + return r1::try_acquire(m, *this); +} + +//! Release lock +inline void rtm_mutex::scoped_lock::release() { + __TBB_ASSERT(m_mutex, "lock is not acquired"); + __TBB_ASSERT(m_transaction_state != rtm_state::rtm_none, "lock is not acquired"); + return r1::release(*this); +} + +#if _MSC_VER && !defined(__INTEL_COMPILER) + #pragma warning (pop) // 4324 warning +#endif + +#if TBB_USE_PROFILING_TOOLS +inline void set_name(rtm_mutex& obj, const char* name) { + itt_set_sync_name(&obj, name); +} +#if (_WIN32||_WIN64) +inline void set_name(rtm_mutex& obj, const wchar_t* name) { + itt_set_sync_name(&obj, name); +} +#endif // WIN +#else +inline void set_name(rtm_mutex&, const char*) {} +#if (_WIN32||_WIN64) +inline void set_name(rtm_mutex&, const wchar_t*) {} +#endif // WIN +#endif + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif /* __TBB__rtm_mutex_impl_H */ diff --git a/src/3rd party/oneapi/tbb/detail/_rtm_rw_mutex.h b/src/3rd party/oneapi/tbb/detail/_rtm_rw_mutex.h new file mode 100644 index 0000000000..702e65e401 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_rtm_rw_mutex.h @@ -0,0 +1,216 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__rtm_rw_mutex_H +#define __TBB_detail__rtm_rw_mutex_H + +#include "_assert.h" +#include "_utils.h" +#include "../spin_rw_mutex.h" + +#include + +namespace tbb { +namespace detail { + +namespace r1 { +struct rtm_rw_mutex_impl; +} + +namespace d1 { + +__TBB_GLOBAL_VAR constexpr std::size_t speculation_granularity = 64; +#if _MSC_VER && !defined(__INTEL_COMPILER) + // Suppress warning: structure was padded due to alignment specifier + #pragma warning (push) + #pragma warning (disable: 4324) +#endif + +//! Fast, unfair, spinning speculation-enabled reader-writer lock with backoff and writer-preference +/** @ingroup synchronization */ +class alignas(max_nfs_size) rtm_rw_mutex : private spin_rw_mutex { + friend struct r1::rtm_rw_mutex_impl; +private: + enum class rtm_type { + rtm_not_in_mutex, + rtm_transacting_reader, + rtm_transacting_writer, + rtm_real_reader, + rtm_real_writer + }; +public: + //! Constructors + rtm_rw_mutex() noexcept : write_flag(false) { + create_itt_sync(this, "tbb::speculative_spin_rw_mutex", ""); + } + + //! Destructor + ~rtm_rw_mutex() = default; + + //! Represents acquisition of a mutex. + class scoped_lock { + friend struct r1::rtm_rw_mutex_impl; + public: + //! Construct lock that has not acquired a mutex. + /** Equivalent to zero-initialization of *this. */ + constexpr scoped_lock() : m_mutex(nullptr), m_transaction_state(rtm_type::rtm_not_in_mutex) {} + + //! Acquire lock on given mutex. + scoped_lock(rtm_rw_mutex& m, bool write = true) : m_mutex(nullptr), m_transaction_state(rtm_type::rtm_not_in_mutex) { + acquire(m, write); + } + + //! Release lock (if lock is held). + ~scoped_lock() { + if(m_transaction_state != rtm_type::rtm_not_in_mutex) { + release(); + } + } + + //! No Copy + scoped_lock(const scoped_lock&) = delete; + scoped_lock& operator=(const scoped_lock&) = delete; + + //! Acquire lock on given mutex. + inline void acquire(rtm_rw_mutex& m, bool write = true); + + //! Try acquire lock on given mutex. + inline bool try_acquire(rtm_rw_mutex& m, bool write = true); + + //! Release lock + inline void release(); + + //! Upgrade reader to become a writer. + /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ + inline bool upgrade_to_writer(); + + //! Downgrade writer to become a reader. + inline bool downgrade_to_reader(); + + inline bool is_writer() const; + private: + rtm_rw_mutex* m_mutex; + rtm_type m_transaction_state; + }; + + //! Mutex traits + static constexpr bool is_rw_mutex = true; + static constexpr bool is_recursive_mutex = false; + static constexpr bool is_fair_mutex = false; + +private: + alignas(speculation_granularity) std::atomic write_flag; +}; + +#if _MSC_VER && !defined(__INTEL_COMPILER) + #pragma warning (pop) // 4324 warning +#endif + +} // namespace d1 + +namespace r1 { + //! Internal acquire write lock. + // only_speculate == true if we're doing a try_lock, else false. + TBB_EXPORT void __TBB_EXPORTED_FUNC acquire_writer(d1::rtm_rw_mutex&, d1::rtm_rw_mutex::scoped_lock&, bool only_speculate = false); + //! Internal acquire read lock. + // only_speculate == true if we're doing a try_lock, else false. + TBB_EXPORT void __TBB_EXPORTED_FUNC acquire_reader(d1::rtm_rw_mutex&, d1::rtm_rw_mutex::scoped_lock&, bool only_speculate = false); + //! Internal upgrade reader to become a writer. + TBB_EXPORT bool __TBB_EXPORTED_FUNC upgrade(d1::rtm_rw_mutex::scoped_lock&); + //! Internal downgrade writer to become a reader. + TBB_EXPORT bool __TBB_EXPORTED_FUNC downgrade(d1::rtm_rw_mutex::scoped_lock&); + //! Internal try_acquire write lock. + TBB_EXPORT bool __TBB_EXPORTED_FUNC try_acquire_writer(d1::rtm_rw_mutex&, d1::rtm_rw_mutex::scoped_lock&); + //! Internal try_acquire read lock. + TBB_EXPORT bool __TBB_EXPORTED_FUNC try_acquire_reader(d1::rtm_rw_mutex&, d1::rtm_rw_mutex::scoped_lock&); + //! Internal release lock. + TBB_EXPORT void __TBB_EXPORTED_FUNC release(d1::rtm_rw_mutex::scoped_lock&); +} + +namespace d1 { +//! Acquire lock on given mutex. +void rtm_rw_mutex::scoped_lock::acquire(rtm_rw_mutex& m, bool write) { + __TBB_ASSERT(!m_mutex, "lock is already acquired"); + if (write) { + r1::acquire_writer(m, *this); + } else { + r1::acquire_reader(m, *this); + } +} + +//! Try acquire lock on given mutex. +bool rtm_rw_mutex::scoped_lock::try_acquire(rtm_rw_mutex& m, bool write) { + __TBB_ASSERT(!m_mutex, "lock is already acquired"); + if (write) { + return r1::try_acquire_writer(m, *this); + } else { + return r1::try_acquire_reader(m, *this); + } +} + +//! Release lock +void rtm_rw_mutex::scoped_lock::release() { + __TBB_ASSERT(m_mutex, "lock is not acquired"); + __TBB_ASSERT(m_transaction_state != rtm_type::rtm_not_in_mutex, "lock is not acquired"); + return r1::release(*this); +} + +//! Upgrade reader to become a writer. +/** Returns whether the upgrade happened without releasing and re-acquiring the lock */ +bool rtm_rw_mutex::scoped_lock::upgrade_to_writer() { + __TBB_ASSERT(m_mutex, "lock is not acquired"); + if (m_transaction_state == rtm_type::rtm_transacting_writer || m_transaction_state == rtm_type::rtm_real_writer) { + return true; // Already a writer + } + return r1::upgrade(*this); +} + +//! Downgrade writer to become a reader. +bool rtm_rw_mutex::scoped_lock::downgrade_to_reader() { + __TBB_ASSERT(m_mutex, "lock is not acquired"); + if (m_transaction_state == rtm_type::rtm_transacting_reader || m_transaction_state == rtm_type::rtm_real_reader) { + return true; // Already a reader + } + return r1::downgrade(*this); +} + +bool rtm_rw_mutex::scoped_lock::is_writer() const { + __TBB_ASSERT(m_mutex, "lock is not acquired"); + return m_transaction_state == rtm_type::rtm_transacting_writer || m_transaction_state == rtm_type::rtm_real_writer; +} + +#if TBB_USE_PROFILING_TOOLS +inline void set_name(rtm_rw_mutex& obj, const char* name) { + itt_set_sync_name(&obj, name); +} +#if (_WIN32||_WIN64) +inline void set_name(rtm_rw_mutex& obj, const wchar_t* name) { + itt_set_sync_name(&obj, name); +} +#endif // WIN +#else +inline void set_name(rtm_rw_mutex&, const char*) {} +#if (_WIN32||_WIN64) +inline void set_name(rtm_rw_mutex&, const wchar_t*) {} +#endif // WIN +#endif + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__rtm_rw_mutex_H diff --git a/src/3rd party/oneapi/tbb/detail/_scoped_lock.h b/src/3rd party/oneapi/tbb/detail/_scoped_lock.h new file mode 100644 index 0000000000..a49dcdff53 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_scoped_lock.h @@ -0,0 +1,174 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail_scoped_lock_H +#define __TBB_detail_scoped_lock_H + +namespace tbb { +namespace detail { +namespace d1 { + +// unique_scoped_lock supposes that Mutex operations never throw +template +class unique_scoped_lock { + //! Points to currently held Mutex, or nullptr if no lock is held. + Mutex* m_mutex{}; + +public: + //! Construct without acquiring a Mutex. + constexpr unique_scoped_lock() noexcept : m_mutex(nullptr) {} + + //! Construct and acquire lock on a Mutex. + unique_scoped_lock(Mutex& m) { + acquire(m); + } + + //! No Copy + unique_scoped_lock(const unique_scoped_lock&) = delete; + unique_scoped_lock& operator=(const unique_scoped_lock&) = delete; + + //! Acquire lock. + void acquire(Mutex& m) { + __TBB_ASSERT(m_mutex == nullptr, "The mutex is already acquired"); + m_mutex = &m; + m.lock(); + } + + //! Try acquiring lock (non-blocking) + /** Return true if lock acquired; false otherwise. */ + bool try_acquire(Mutex& m) { + __TBB_ASSERT(m_mutex == nullptr, "The mutex is already acquired"); + bool succeed = m.try_lock(); + if (succeed) { + m_mutex = &m; + } + return succeed; + } + + //! Release lock + void release() { + __TBB_ASSERT(m_mutex, "release on Mutex::unique_scoped_lock that is not holding a lock"); + m_mutex->unlock(); + m_mutex = nullptr; + } + + //! Destroy lock. If holding a lock, releases the lock first. + ~unique_scoped_lock() { + if (m_mutex) { + release(); + } + } +}; + +// rw_scoped_lock supposes that Mutex operations never throw +template +class rw_scoped_lock { +public: + //! Construct lock that has not acquired a mutex. + /** Equivalent to zero-initialization of *this. */ + constexpr rw_scoped_lock() noexcept {} + + //! Acquire lock on given mutex. + rw_scoped_lock(Mutex& m, bool write = true) { + acquire(m, write); + } + + //! Release lock (if lock is held). + ~rw_scoped_lock() { + if (m_mutex) { + release(); + } + } + + //! No Copy + rw_scoped_lock(const rw_scoped_lock&) = delete; + rw_scoped_lock& operator=(const rw_scoped_lock&) = delete; + + //! Acquire lock on given mutex. + void acquire(Mutex& m, bool write = true) { + __TBB_ASSERT(m_mutex == nullptr, "The mutex is already acquired"); + m_is_writer = write; + m_mutex = &m; + if (write) { + m_mutex->lock(); + } else { + m_mutex->lock_shared(); + } + } + + //! Try acquire lock on given mutex. + bool try_acquire(Mutex& m, bool write = true) { + bool succeed = write ? m.try_lock() : m.try_lock_shared(); + if (succeed) { + m_mutex = &m; + m_is_writer = write; + } + return succeed; + } + + //! Release lock. + void release() { + __TBB_ASSERT(m_mutex != nullptr, "The mutex is not acquired"); + Mutex* m = m_mutex; + m_mutex = nullptr; + + if (m_is_writer) { + m->unlock(); + } else { + m->unlock_shared(); + } + } + + //! Upgrade reader to become a writer. + /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ + bool upgrade_to_writer() { + __TBB_ASSERT(m_mutex != nullptr, "The mutex is not acquired"); + if (m_is_writer) { + return true; // Already a writer + } + m_is_writer = true; + return m_mutex->upgrade(); + } + + //! Downgrade writer to become a reader. + bool downgrade_to_reader() { + __TBB_ASSERT(m_mutex != nullptr, "The mutex is not acquired"); + if (m_is_writer) { + m_mutex->downgrade(); + m_is_writer = false; + } + return true; + } + + bool is_writer() const { + __TBB_ASSERT(m_mutex != nullptr, "The mutex is not acquired"); + return m_is_writer; + } + +protected: + //! The pointer to the current mutex that is held, or nullptr if no mutex is held. + Mutex* m_mutex {nullptr}; + + //! If mutex != nullptr, then is_writer is true if holding a writer lock, false if holding a reader lock. + /** Not defined if not holding a lock. */ + bool m_is_writer {false}; +}; + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail_scoped_lock_H diff --git a/src/3rd party/oneapi/tbb/detail/_segment_table.h b/src/3rd party/oneapi/tbb/detail/_segment_table.h new file mode 100644 index 0000000000..7fbf1cc0f9 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_segment_table.h @@ -0,0 +1,581 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__segment_table_H +#define __TBB_detail__segment_table_H + +#include "_config.h" +#include "_allocator_traits.h" +#include "_template_helpers.h" +#include "_utils.h" +#include "_assert.h" +#include "_exception.h" +#include +#include +#include +#include + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(push) +#pragma warning(disable: 4127) // warning C4127: conditional expression is constant +#endif + +namespace tbb { +namespace detail { +namespace d1 { + +template +class segment_table { +public: + using value_type = T; + using segment_type = T*; + using atomic_segment = std::atomic; + using segment_table_type = atomic_segment*; + + using size_type = std::size_t; + using segment_index_type = std::size_t; + + using allocator_type = Allocator; + + using allocator_traits_type = tbb::detail::allocator_traits; + using segment_table_allocator_type = typename allocator_traits_type::template rebind_alloc; +protected: + using segment_table_allocator_traits = tbb::detail::allocator_traits; + using derived_type = DerivedType; + + static constexpr size_type pointers_per_embedded_table = PointersPerEmbeddedTable; + static constexpr size_type pointers_per_long_table = sizeof(size_type) * 8; +public: + segment_table( const allocator_type& alloc = allocator_type() ) + : my_segment_table_allocator(alloc), my_segment_table(nullptr) + , my_first_block{}, my_size{}, my_segment_table_allocation_failed{} + { + my_segment_table.store(my_embedded_table, std::memory_order_relaxed); + zero_table(my_embedded_table, pointers_per_embedded_table); + } + + segment_table( const segment_table& other ) + : my_segment_table_allocator(segment_table_allocator_traits:: + select_on_container_copy_construction(other.my_segment_table_allocator)) + , my_segment_table(nullptr), my_first_block{}, my_size{}, my_segment_table_allocation_failed{} + { + my_segment_table.store(my_embedded_table, std::memory_order_relaxed); + zero_table(my_embedded_table, pointers_per_embedded_table); + try_call( [&] { + internal_transfer(other, copy_segment_body_type{*this}); + } ).on_exception( [&] { + clear(); + }); + } + + segment_table( const segment_table& other, const allocator_type& alloc ) + : my_segment_table_allocator(alloc), my_segment_table(nullptr) + , my_first_block{}, my_size{}, my_segment_table_allocation_failed{} + { + my_segment_table.store(my_embedded_table, std::memory_order_relaxed); + zero_table(my_embedded_table, pointers_per_embedded_table); + try_call( [&] { + internal_transfer(other, copy_segment_body_type{*this}); + } ).on_exception( [&] { + clear(); + }); + } + + segment_table( segment_table&& other ) + : my_segment_table_allocator(std::move(other.my_segment_table_allocator)), my_segment_table(nullptr) + , my_first_block{}, my_size{}, my_segment_table_allocation_failed{} + { + my_segment_table.store(my_embedded_table, std::memory_order_relaxed); + zero_table(my_embedded_table, pointers_per_embedded_table); + internal_move(std::move(other)); + } + + segment_table( segment_table&& other, const allocator_type& alloc ) + : my_segment_table_allocator(alloc), my_segment_table(nullptr), my_first_block{} + , my_size{}, my_segment_table_allocation_failed{} + { + my_segment_table.store(my_embedded_table, std::memory_order_relaxed); + zero_table(my_embedded_table, pointers_per_embedded_table); + using is_equal_type = typename segment_table_allocator_traits::is_always_equal; + internal_move_construct_with_allocator(std::move(other), alloc, is_equal_type()); + } + + ~segment_table() { + clear(); + } + + segment_table& operator=( const segment_table& other ) { + if (this != &other) { + copy_assign_allocators(my_segment_table_allocator, other.my_segment_table_allocator); + internal_transfer(other, copy_segment_body_type{*this}); + } + return *this; + } + + segment_table& operator=( segment_table&& other ) + noexcept(derived_type::is_noexcept_assignment) + { + using pocma_type = typename segment_table_allocator_traits::propagate_on_container_move_assignment; + using is_equal_type = typename segment_table_allocator_traits::is_always_equal; + + if (this != &other) { + move_assign_allocators(my_segment_table_allocator, other.my_segment_table_allocator); + internal_move_assign(std::move(other), tbb::detail::disjunction()); + } + return *this; + } + + void swap( segment_table& other ) + noexcept(derived_type::is_noexcept_swap) + { + using is_equal_type = typename segment_table_allocator_traits::is_always_equal; + using pocs_type = typename segment_table_allocator_traits::propagate_on_container_swap; + + if (this != &other) { + swap_allocators(my_segment_table_allocator, other.my_segment_table_allocator); + internal_swap(other, tbb::detail::disjunction()); + } + } + + segment_type get_segment( segment_index_type index ) const { + return get_table()[index] + segment_base(index); + } + + value_type& operator[]( size_type index ) { + return internal_subscript(index); + } + + const value_type& operator[]( size_type index ) const { + return const_cast(this)->internal_subscript(index); + } + + const segment_table_allocator_type& get_allocator() const { + return my_segment_table_allocator; + } + + segment_table_allocator_type& get_allocator() { + return my_segment_table_allocator; + } + + void enable_segment( segment_type& segment, segment_table_type table, segment_index_type seg_index, size_type index ) { + // Allocate new segment + segment_type new_segment = self()->create_segment(table, seg_index, index); + if (new_segment != nullptr) { + // Store (new_segment - segment_base) into the segment table to allow access to the table by index via + // my_segment_table[segment_index_of(index)][index] + segment_type disabled_segment = nullptr; + if (!table[seg_index].compare_exchange_strong(disabled_segment, new_segment - segment_base(seg_index))) { + // compare_exchange failed => some other thread has already enabled this segment + // Deallocate the memory + self()->deallocate_segment(new_segment, seg_index); + } + } + + segment = table[seg_index].load(std::memory_order_acquire); + __TBB_ASSERT(segment != nullptr, "If create_segment returned nullptr, the element should be stored in the table"); + } + + void delete_segment( segment_index_type seg_index ) { + segment_type segment_to_delete = self()->nullify_segment(get_table(), seg_index); + if (segment_to_delete == segment_allocation_failure_tag) { + return; + } + + segment_to_delete += segment_base(seg_index); + + // Deallocate the segment + self()->destroy_segment(segment_to_delete, seg_index); + } + + size_type number_of_segments( segment_table_type table ) const { + // Check for an active table, if it is embedded table - return the number of embedded segments + // Otherwise - return the maximum number of segments + return table == my_embedded_table ? pointers_per_embedded_table : pointers_per_long_table; + } + + size_type capacity() const noexcept { + segment_table_type table = get_table(); + size_type num_segments = number_of_segments(table); + for (size_type seg_index = 0; seg_index < num_segments; ++seg_index) { + // Check if the pointer is valid (allocated) + if (table[seg_index].load(std::memory_order_relaxed) <= segment_allocation_failure_tag) { + return segment_base(seg_index); + } + } + return segment_base(num_segments); + } + + size_type find_last_allocated_segment( segment_table_type table ) const noexcept { + size_type end = 0; + size_type num_segments = number_of_segments(table); + for (size_type seg_index = 0; seg_index < num_segments; ++seg_index) { + // Check if the pointer is valid (allocated) + if (table[seg_index].load(std::memory_order_relaxed) > segment_allocation_failure_tag) { + end = seg_index + 1; + } + } + return end; + } + + void reserve( size_type n ) { + if (n > allocator_traits_type::max_size(my_segment_table_allocator)) { + throw_exception(exception_id::reservation_length_error); + } + + size_type size = my_size.load(std::memory_order_relaxed); + segment_index_type start_seg_idx = size == 0 ? 0 : segment_index_of(size - 1) + 1; + for (segment_index_type seg_idx = start_seg_idx; segment_base(seg_idx) < n; ++seg_idx) { + size_type first_index = segment_base(seg_idx); + internal_subscript(first_index); + } + } + + void clear() { + clear_segments(); + clear_table(); + my_size.store(0, std::memory_order_relaxed); + my_first_block.store(0, std::memory_order_relaxed); + } + + void clear_segments() { + segment_table_type current_segment_table = get_table(); + for (size_type i = number_of_segments(current_segment_table); i != 0; --i) { + if (current_segment_table[i - 1].load(std::memory_order_relaxed) != nullptr) { + // If the segment was enabled - disable and deallocate it + delete_segment(i - 1); + } + } + } + + void destroy_and_deallocate_table(segment_table_type table, size_type num_segments) { + auto& alloc = get_allocator(); + for (size_type seg_idx = 0; seg_idx < num_segments; ++seg_idx) { + segment_table_allocator_traits::destroy(alloc, &table[seg_idx]); + } + segment_table_allocator_traits::deallocate(alloc, table, num_segments); + } + + void clear_table() { + segment_table_type current_segment_table = get_table(); + if (current_segment_table != my_embedded_table) { + // If the active table is not the embedded one - deallocate the active table + destroy_and_deallocate_table(current_segment_table, pointers_per_long_table); + my_segment_table.store(my_embedded_table, std::memory_order_relaxed); + zero_table(my_embedded_table, pointers_per_embedded_table); + } + } + + void extend_table_if_necessary(segment_table_type& table, size_type start_index, size_type end_index) { + // Extend segment table if an active table is an embedded one and the requested index is + // outside it + if (table == my_embedded_table && end_index > embedded_table_size) { + if (start_index <= embedded_table_size) { + // More than one thread can get here: the one that has assigned the first block and + // is in the process of allocating it now, and the one that saw the first block has + // been assigned already, but not yet allocated. This latter thread decides not to + // wait for the first one and extend the table itself. + try_call([&] { + segment_table_type new_table = + self()->allocate_long_table(my_embedded_table, start_index); + // It is possible that the table was extended by the thread that allocated first + // block. In this case, the below CAS fails and re-reads the new table pointer. + if (my_segment_table.compare_exchange_strong( + table, new_table, + /*memory order in case of a success*/std::memory_order_release, + /*memory order in case of a failure*/std::memory_order_acquire)) + { + // CAS was successful, update the local table pointer with now actual + table = new_table; + } else if (new_table) { + // Other thread was the first to replace the segment table. Current thread's + // table is not needed anymore, so destroying it. + destroy_and_deallocate_table(new_table, pointers_per_long_table); + } + }).on_exception([&] { + my_segment_table_allocation_failed.store(true, std::memory_order_relaxed); + }); + } else { + atomic_backoff backoff; + do { + if (my_segment_table_allocation_failed.load(std::memory_order_relaxed)) { + throw_exception(exception_id::bad_alloc); + } + backoff.pause(); + table = my_segment_table.load(std::memory_order_acquire); + } while (table == my_embedded_table); + } + } + } + + // Return the segment where index is stored + static constexpr segment_index_type segment_index_of( size_type index ) { + return size_type(tbb::detail::log2(uintptr_t(index|1))); + } + + // Needed to calculate the offset in segment + static constexpr size_type segment_base( size_type index ) { + return size_type(1) << index & ~size_type(1); + } + + // Return size of the segment + static constexpr size_type segment_size( size_type index ) { + return index == 0 ? 2 : size_type(1) << index; + } + +private: + + derived_type* self() { + return static_cast(this); + } + + struct copy_segment_body_type { + void operator()( segment_index_type index, segment_type from, segment_type to ) const { + my_instance.self()->copy_segment(index, from, to); + } + segment_table& my_instance; + }; + + struct move_segment_body_type { + void operator()( segment_index_type index, segment_type from, segment_type to ) const { + my_instance.self()->move_segment(index, from, to); + } + segment_table& my_instance; + }; + + // Transgers all segments from the other table + template + void internal_transfer( const segment_table& other, TransferBody transfer_segment ) { + static_cast(this)->destroy_elements(); + + assign_first_block_if_necessary(other.my_first_block.load(std::memory_order_relaxed)); + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + + segment_table_type other_table = other.get_table(); + size_type end_segment_size = segment_size(other.find_last_allocated_segment(other_table)); + + // If an exception occurred in other, then the size may be greater than the size of the end segment. + size_type other_size = end_segment_size < other.my_size.load(std::memory_order_relaxed) ? + other.my_size.load(std::memory_order_relaxed) : end_segment_size; + other_size = my_segment_table_allocation_failed ? embedded_table_size : other_size; + + for (segment_index_type i = 0; segment_base(i) < other_size; ++i) { + // If the segment in other table is enabled - transfer it + if (other_table[i].load(std::memory_order_relaxed) == segment_allocation_failure_tag) + { + my_size = segment_base(i); + break; + } else if (other_table[i].load(std::memory_order_relaxed) != nullptr) { + internal_subscript(segment_base(i)); + transfer_segment(i, other.get_table()[i].load(std::memory_order_relaxed) + segment_base(i), + get_table()[i].load(std::memory_order_relaxed) + segment_base(i)); + } + } + } + + // Moves the other segment table + // Only equal allocators are allowed + void internal_move( segment_table&& other ) { + // NOTE: allocators should be equal + clear(); + my_first_block.store(other.my_first_block.load(std::memory_order_relaxed), std::memory_order_relaxed); + my_size.store(other.my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + // If an active table in other is embedded - restore all of the embedded segments + if (other.get_table() == other.my_embedded_table) { + for ( size_type i = 0; i != pointers_per_embedded_table; ++i ) { + segment_type other_segment = other.my_embedded_table[i].load(std::memory_order_relaxed); + my_embedded_table[i].store(other_segment, std::memory_order_relaxed); + other.my_embedded_table[i].store(nullptr, std::memory_order_relaxed); + } + my_segment_table.store(my_embedded_table, std::memory_order_relaxed); + } else { + my_segment_table.store(other.my_segment_table, std::memory_order_relaxed); + other.my_segment_table.store(other.my_embedded_table, std::memory_order_relaxed); + zero_table(other.my_embedded_table, pointers_per_embedded_table); + } + other.my_size.store(0, std::memory_order_relaxed); + } + + // Move construct the segment table with the allocator object + // if any instances of allocator_type are always equal + void internal_move_construct_with_allocator( segment_table&& other, const allocator_type&, + /*is_always_equal = */ std::true_type ) { + internal_move(std::move(other)); + } + + // Move construct the segment table with the allocator object + // if any instances of allocator_type are always equal + void internal_move_construct_with_allocator( segment_table&& other, const allocator_type& alloc, + /*is_always_equal = */ std::false_type ) { + if (other.my_segment_table_allocator == alloc) { + // If allocators are equal - restore pointers + internal_move(std::move(other)); + } else { + // If allocators are not equal - perform per element move with reallocation + try_call( [&] { + internal_transfer(other, move_segment_body_type{*this}); + } ).on_exception( [&] { + clear(); + }); + } + } + + // Move assigns the segment table to other is any instances of allocator_type are always equal + // or propagate_on_container_move_assignment is true + void internal_move_assign( segment_table&& other, /*is_always_equal || POCMA = */ std::true_type ) { + internal_move(std::move(other)); + } + + // Move assigns the segment table to other is any instances of allocator_type are not always equal + // and propagate_on_container_move_assignment is false + void internal_move_assign( segment_table&& other, /*is_always_equal || POCMA = */ std::false_type ) { + if (my_segment_table_allocator == other.my_segment_table_allocator) { + // If allocators are equal - restore pointers + internal_move(std::move(other)); + } else { + // If allocators are not equal - perform per element move with reallocation + internal_transfer(other, move_segment_body_type{*this}); + } + } + + // Swaps two segment tables if any instances of allocator_type are always equal + // or propagate_on_container_swap is true + void internal_swap( segment_table& other, /*is_always_equal || POCS = */ std::true_type ) { + internal_swap_fields(other); + } + + // Swaps two segment tables if any instances of allocator_type are not always equal + // and propagate_on_container_swap is false + // According to the C++ standard, swapping of two containers with unequal allocators + // is an undefined behavior scenario + void internal_swap( segment_table& other, /*is_always_equal || POCS = */ std::false_type ) { + __TBB_ASSERT(my_segment_table_allocator == other.my_segment_table_allocator, + "Swapping with unequal allocators is not allowed"); + internal_swap_fields(other); + } + + void internal_swap_fields( segment_table& other ) { + // If an active table in either *this segment table or other is an embedded one - swaps the embedded tables + if (get_table() == my_embedded_table || + other.get_table() == other.my_embedded_table) { + + for (size_type i = 0; i != pointers_per_embedded_table; ++i) { + segment_type current_segment = my_embedded_table[i].load(std::memory_order_relaxed); + segment_type other_segment = other.my_embedded_table[i].load(std::memory_order_relaxed); + + my_embedded_table[i].store(other_segment, std::memory_order_relaxed); + other.my_embedded_table[i].store(current_segment, std::memory_order_relaxed); + } + } + + segment_table_type current_segment_table = get_table(); + segment_table_type other_segment_table = other.get_table(); + + // If an active table is an embedded one - + // store an active table in other to the embedded one from other + if (current_segment_table == my_embedded_table) { + other.my_segment_table.store(other.my_embedded_table, std::memory_order_relaxed); + } else { + // Otherwise - store it to the active segment table + other.my_segment_table.store(current_segment_table, std::memory_order_relaxed); + } + + // If an active table in other segment table is an embedded one - + // store an active table in other to the embedded one from *this + if (other_segment_table == other.my_embedded_table) { + my_segment_table.store(my_embedded_table, std::memory_order_relaxed); + } else { + // Otherwise - store it to the active segment table in other + my_segment_table.store(other_segment_table, std::memory_order_relaxed); + } + auto first_block = other.my_first_block.load(std::memory_order_relaxed); + other.my_first_block.store(my_first_block.load(std::memory_order_relaxed), std::memory_order_relaxed); + my_first_block.store(first_block, std::memory_order_relaxed); + + auto size = other.my_size.load(std::memory_order_relaxed); + other.my_size.store(my_size.load(std::memory_order_relaxed), std::memory_order_relaxed); + my_size.store(size, std::memory_order_relaxed); + } + +protected: + // A flag indicates that an exception was throws during segment allocations + const segment_type segment_allocation_failure_tag = reinterpret_cast(1); + static constexpr size_type embedded_table_size = segment_size(pointers_per_embedded_table); + + template + value_type& internal_subscript( size_type index ) { + segment_index_type seg_index = segment_index_of(index); + segment_table_type table = my_segment_table.load(std::memory_order_acquire); + segment_type segment = nullptr; + + if (allow_out_of_range_access) { + if (derived_type::allow_table_extending) { + extend_table_if_necessary(table, index, index + 1); + } + + segment = table[seg_index].load(std::memory_order_acquire); + // If the required segment is disabled - enable it + if (segment == nullptr) { + enable_segment(segment, table, seg_index, index); + } + // Check if an exception was thrown during segment allocation + if (segment == segment_allocation_failure_tag) { + throw_exception(exception_id::bad_alloc); + } + } else { + segment = table[seg_index].load(std::memory_order_acquire); + } + __TBB_ASSERT(segment != nullptr, nullptr); + + return segment[index]; + } + + void assign_first_block_if_necessary(segment_index_type index) { + size_type zero = 0; + if (this->my_first_block.load(std::memory_order_relaxed) == zero) { + this->my_first_block.compare_exchange_strong(zero, index); + } + } + + void zero_table( segment_table_type table, size_type count ) { + for (size_type i = 0; i != count; ++i) { + table[i].store(nullptr, std::memory_order_relaxed); + } + } + + segment_table_type get_table() const { + return my_segment_table.load(std::memory_order_acquire); + } + + segment_table_allocator_type my_segment_table_allocator; + std::atomic my_segment_table; + atomic_segment my_embedded_table[pointers_per_embedded_table]; + // Number of segments in first block + std::atomic my_first_block; + // Number of elements in table + std::atomic my_size; + // Flag to indicate failed extend table + std::atomic my_segment_table_allocation_failed; +}; // class segment_table + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) +#pragma warning(pop) // warning 4127 is back +#endif + +#endif // __TBB_detail__segment_table_H diff --git a/src/3rd party/oneapi/tbb/detail/_small_object_pool.h b/src/3rd party/oneapi/tbb/detail/_small_object_pool.h new file mode 100644 index 0000000000..7485b31c76 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_small_object_pool.h @@ -0,0 +1,108 @@ +/* + Copyright (c) 2020-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__small_object_pool_H +#define __TBB__small_object_pool_H + +#include "_config.h" +#include "_assert.h" + +#include "../profiling.h" +#include +#include +#include + +namespace tbb { +namespace detail { + +namespace d1 { +class small_object_pool { +protected: + small_object_pool() = default; +}; +struct execution_data; +} + +namespace r1 { +TBB_EXPORT void* __TBB_EXPORTED_FUNC allocate(d1::small_object_pool*& pool, std::size_t number_of_bytes, + const d1::execution_data& ed); +TBB_EXPORT void* __TBB_EXPORTED_FUNC allocate(d1::small_object_pool*& pool, std::size_t number_of_bytes); +TBB_EXPORT void __TBB_EXPORTED_FUNC deallocate(d1::small_object_pool& pool, void* ptr, std::size_t number_of_bytes, + const d1::execution_data& ed); +TBB_EXPORT void __TBB_EXPORTED_FUNC deallocate(d1::small_object_pool& pool, void* ptr, std::size_t number_of_bytes); +} + +namespace d1 { +class small_object_allocator { +public: + template + Type* new_object(execution_data& ed, Args&&... args) { + void* allocated_object = r1::allocate(m_pool, sizeof(Type), ed); + + auto constructed_object = new(allocated_object) Type(std::forward(args)...); + return constructed_object; + } + + template + Type* new_object(Args&&... args) { + void* allocated_object = r1::allocate(m_pool, sizeof(Type)); + + auto constructed_object = new(allocated_object) Type(std::forward(args)...); + return constructed_object; + } + + template + void delete_object(Type* object, const execution_data& ed) { + // Copy this since it can be a member of the passed object and + // unintentionally destroyed when Type destructor is called below + small_object_allocator alloc = *this; + object->~Type(); + alloc.deallocate(object, ed); + } + + template + void delete_object(Type* object) { + // Copy this since it can be a member of the passed object and + // unintentionally destroyed when Type destructor is called below + small_object_allocator alloc = *this; + object->~Type(); + alloc.deallocate(object); + } + + template + void deallocate(Type* ptr, const execution_data& ed) { + call_itt_task_notify(destroy, ptr); + + __TBB_ASSERT(m_pool != nullptr, "Pool must be valid for deallocate call"); + r1::deallocate(*m_pool, ptr, sizeof(Type), ed); + } + + template + void deallocate(Type* ptr) { + call_itt_task_notify(destroy, ptr); + + __TBB_ASSERT(m_pool != nullptr, "Pool must be valid for deallocate call"); + r1::deallocate(*m_pool, ptr, sizeof(Type)); + } +private: + small_object_pool* m_pool{}; +}; + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif /* __TBB__small_object_pool_H */ diff --git a/src/3rd party/tbb/internal/_tbb_strings.h b/src/3rd party/oneapi/tbb/detail/_string_resource.h similarity index 86% rename from src/3rd party/tbb/internal/_tbb_strings.h rename to src/3rd party/oneapi/tbb/detail/_string_resource.h index ee00525bba..c06d5b5db0 100644 --- a/src/3rd party/tbb/internal/_tbb_strings.h +++ b/src/3rd party/oneapi/tbb/detail/_string_resource.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,6 +14,17 @@ limitations under the License. */ +TBB_STRING_RESOURCE(ALGORITHM, "tbb_algorithm") +TBB_STRING_RESOURCE(PARALLEL_FOR, "tbb_parallel_for") +TBB_STRING_RESOURCE(PARALLEL_FOR_EACH, "tbb_parallel_for_each") +TBB_STRING_RESOURCE(PARALLEL_INVOKE, "tbb_parallel_invoke") +TBB_STRING_RESOURCE(PARALLEL_REDUCE, "tbb_parallel_reduce") +TBB_STRING_RESOURCE(PARALLEL_SCAN, "tbb_parallel_scan") +TBB_STRING_RESOURCE(PARALLEL_SORT, "tbb_parallel_sort") +TBB_STRING_RESOURCE(PARALLEL_PIPELINE, "tbb_parallel_pipeline") +TBB_STRING_RESOURCE(CUSTOM_CTX, "tbb_custom") + +TBB_STRING_RESOURCE(FLOW_NULL, "null") TBB_STRING_RESOURCE(FLOW_BROADCAST_NODE, "broadcast_node") TBB_STRING_RESOURCE(FLOW_BUFFER_NODE, "buffer_node") TBB_STRING_RESOURCE(FLOW_CONTINUE_NODE, "continue_node") @@ -23,17 +34,16 @@ TBB_STRING_RESOURCE(FLOW_JOIN_NODE_RESERVING, "join_node (reserving)") TBB_STRING_RESOURCE(FLOW_JOIN_NODE_TAG_MATCHING, "join_node (tag_matching)") TBB_STRING_RESOURCE(FLOW_LIMITER_NODE, "limiter_node") TBB_STRING_RESOURCE(FLOW_MULTIFUNCTION_NODE, "multifunction_node") -TBB_STRING_RESOURCE(FLOW_OR_NODE, "or_node") //no longer in use, kept for backward compatibility TBB_STRING_RESOURCE(FLOW_OVERWRITE_NODE, "overwrite_node") TBB_STRING_RESOURCE(FLOW_PRIORITY_QUEUE_NODE, "priority_queue_node") TBB_STRING_RESOURCE(FLOW_QUEUE_NODE, "queue_node") TBB_STRING_RESOURCE(FLOW_SEQUENCER_NODE, "sequencer_node") -TBB_STRING_RESOURCE(FLOW_SOURCE_NODE, "source_node") +TBB_STRING_RESOURCE(FLOW_INPUT_NODE, "input_node") TBB_STRING_RESOURCE(FLOW_SPLIT_NODE, "split_node") TBB_STRING_RESOURCE(FLOW_WRITE_ONCE_NODE, "write_once_node") -TBB_STRING_RESOURCE(FLOW_BODY, "body") -TBB_STRING_RESOURCE(FLOW_GRAPH, "graph") -TBB_STRING_RESOURCE(FLOW_NODE, "node") +TBB_STRING_RESOURCE(FLOW_INDEXER_NODE, "indexer_node") +TBB_STRING_RESOURCE(FLOW_COMPOSITE_NODE, "composite_node") +TBB_STRING_RESOURCE(FLOW_ASYNC_NODE, "async_node") TBB_STRING_RESOURCE(FLOW_INPUT_PORT, "input_port") TBB_STRING_RESOURCE(FLOW_INPUT_PORT_0, "input_port_0") TBB_STRING_RESOURCE(FLOW_INPUT_PORT_1, "input_port_1") @@ -57,23 +67,12 @@ TBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_7, "output_port_7") TBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_8, "output_port_8") TBB_STRING_RESOURCE(FLOW_OUTPUT_PORT_9, "output_port_9") TBB_STRING_RESOURCE(FLOW_OBJECT_NAME, "object_name") -TBB_STRING_RESOURCE(FLOW_NULL, "null") -TBB_STRING_RESOURCE(FLOW_INDEXER_NODE, "indexer_node") -TBB_STRING_RESOURCE(FLOW_COMPOSITE_NODE, "composite_node") -TBB_STRING_RESOURCE(FLOW_ASYNC_NODE, "async_node") -TBB_STRING_RESOURCE(FLOW_OPENCL_NODE, "opencl_node") -TBB_STRING_RESOURCE(ALGORITHM, "tbb_algorithm") -TBB_STRING_RESOURCE(PARALLEL_FOR, "tbb_parallel_for") -TBB_STRING_RESOURCE(PARALLEL_DO, "tbb_parallel_do") -TBB_STRING_RESOURCE(PARALLEL_INVOKE, "tbb_parallel_invoke") -TBB_STRING_RESOURCE(PARALLEL_REDUCE, "tbb_parallel_reduce") -TBB_STRING_RESOURCE(PARALLEL_SCAN, "tbb_parallel_scan") -TBB_STRING_RESOURCE(PARALLEL_SORT, "tbb_parallel_sort") -TBB_STRING_RESOURCE(CUSTOM_CTX, "tbb_custom") +TBB_STRING_RESOURCE(FLOW_BODY, "body") +TBB_STRING_RESOURCE(FLOW_GRAPH, "graph") +TBB_STRING_RESOURCE(FLOW_NODE, "node") TBB_STRING_RESOURCE(FLOW_TASKS, "tbb_flow_graph") -TBB_STRING_RESOURCE(PARALLEL_FOR_TASK, "tbb_parallel_for_task") -// TODO: Drop following string prefix "fgt_" here and in FGA's collector -TBB_STRING_RESOURCE(USER_EVENT, "fgt_user_event") -#if __TBB_CPF_BUILD || (TBB_PREVIEW_FLOW_GRAPH_TRACE && TBB_USE_THREADING_TOOLS) +TBB_STRING_RESOURCE(USER_EVENT, "user_event") + +#if __TBB_FLOW_TRACE_CODEPTR TBB_STRING_RESOURCE(CODE_ADDRESS, "code_address") #endif diff --git a/src/3rd party/oneapi/tbb/detail/_task.h b/src/3rd party/oneapi/tbb/detail/_task.h new file mode 100644 index 0000000000..e5bf3d1b2b --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_task.h @@ -0,0 +1,277 @@ +/* + Copyright (c) 2020-2025 Intel Corporation + Copyright (c) 2025 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB__task_H +#define __TBB__task_H + +#include "_config.h" +#include "_assert.h" +#include "_template_helpers.h" +#include "_small_object_pool.h" + +#include "../profiling.h" + +#include +#include +#include +#include +#include +#include + +namespace tbb { +namespace detail { + +namespace d1 { +using slot_id = unsigned short; +__TBB_GLOBAL_VAR constexpr slot_id no_slot = slot_id(~0); +__TBB_GLOBAL_VAR constexpr slot_id any_slot = slot_id(~1); + +class task; +class wait_context; +class task_group_context; +struct execution_data; +class wait_tree_vertex_interface; +class task_arena_base; +} + +namespace d2 { +class task_group; +class task_group_base; +} + +namespace r1 { +//! Task spawn/wait entry points +TBB_EXPORT void __TBB_EXPORTED_FUNC spawn(d1::task& t, d1::task_group_context& ctx); +TBB_EXPORT void __TBB_EXPORTED_FUNC spawn(d1::task& t, d1::task_group_context& ctx, d1::slot_id id); +TBB_EXPORT void __TBB_EXPORTED_FUNC execute_and_wait(d1::task& t, d1::task_group_context& t_ctx, d1::wait_context&, d1::task_group_context& w_ctx); +TBB_EXPORT void __TBB_EXPORTED_FUNC wait(d1::wait_context&, d1::task_group_context& ctx); +TBB_EXPORT d1::slot_id __TBB_EXPORTED_FUNC execution_slot(const d1::execution_data*); +TBB_EXPORT d1::task_group_context* __TBB_EXPORTED_FUNC current_context(); +TBB_EXPORT d1::wait_tree_vertex_interface* get_thread_reference_vertex(d1::wait_tree_vertex_interface* wc); +TBB_EXPORT d1::task* __TBB_EXPORTED_FUNC current_task_ptr(); + +// Do not place under __TBB_RESUMABLE_TASKS. It is a stub for unsupported platforms. +struct suspend_point_type; +using suspend_callback_type = void(*)(void*, suspend_point_type*); +//! The resumable tasks entry points +TBB_EXPORT void __TBB_EXPORTED_FUNC suspend(suspend_callback_type suspend_callback, void* user_callback); +TBB_EXPORT void __TBB_EXPORTED_FUNC resume(suspend_point_type* tag); +TBB_EXPORT suspend_point_type* __TBB_EXPORTED_FUNC current_suspend_point(); +TBB_EXPORT void __TBB_EXPORTED_FUNC notify_waiters(std::uintptr_t wait_ctx_addr); + +class thread_data; +class task_dispatcher; +class external_waiter; +struct task_accessor; +struct task_arena_impl; +} // namespace r1 + +namespace d1 { + +class task_arena; +using suspend_point = r1::suspend_point_type*; + +#if __TBB_RESUMABLE_TASKS +template +inline void suspend_callback(void* user_callback, suspend_point sp) { + // Copy user function to a new stack after the context switch to avoid a race when the previous + // suspend point is resumed while the user_callback is being called. + F user_callback_copy = *static_cast(user_callback); + user_callback_copy(sp); +} + +template +void suspend(F f) { + r1::suspend(&suspend_callback, &f); +} + +inline void resume(suspend_point tag) { + r1::resume(tag); +} +#endif /* __TBB_RESUMABLE_TASKS */ + +// TODO align wait_context on cache lane +class wait_context { + static constexpr std::uint64_t overflow_mask = ~((1LLU << 32) - 1); + + std::uint64_t m_version_and_traits{1}; + std::atomic m_ref_count{}; + + void add_reference(std::int64_t delta) { + call_itt_task_notify(releasing, this); + std::uint64_t r = m_ref_count.fetch_add(static_cast(delta)) + static_cast(delta); + + __TBB_ASSERT_EX((r & overflow_mask) == 0, "Overflow is detected"); + + if (!r) { + // Some external waiters or coroutine waiters sleep in wait list + // Should to notify them that work is done + std::uintptr_t wait_ctx_addr = std::uintptr_t(this); + r1::notify_waiters(wait_ctx_addr); + } + } + + bool continue_execution() const { + std::uint64_t r = m_ref_count.load(std::memory_order_acquire); + __TBB_ASSERT_EX((r & overflow_mask) == 0, "Overflow is detected"); + return r > 0; + } + + friend class r1::thread_data; + friend class r1::task_dispatcher; + friend class r1::external_waiter; + friend class wait_context_vertex; + friend struct r1::task_arena_impl; + friend struct r1::suspend_point_type; +public: + // Despite the internal reference count is uin64_t we limit the user interface with uint32_t + // to preserve a part of the internal reference count for special needs. + wait_context(std::uint32_t ref_count) : m_ref_count{ref_count} { suppress_unused_warning(m_version_and_traits); } + wait_context(const wait_context&) = delete; + + ~wait_context() { + __TBB_ASSERT(!continue_execution(), nullptr); + } + + void reserve(std::uint32_t delta = 1) { + add_reference(delta); + } + + void release(std::uint32_t delta = 1) { + add_reference(-std::int64_t(delta)); + } +}; + +class wait_tree_vertex_interface { +public: + virtual void reserve(std::uint32_t delta = 1) = 0; + virtual void release(std::uint32_t delta = 1) = 0; + +protected: + virtual ~wait_tree_vertex_interface() = default; +}; + +class wait_context_vertex : public wait_tree_vertex_interface { +public: + wait_context_vertex(std::uint32_t ref = 0) : m_wait(ref) {} + + void reserve(std::uint32_t delta = 1) override { + m_wait.reserve(delta); + } + + void release(std::uint32_t delta = 1) override { + m_wait.release(delta); + } + + wait_context& get_context() { + return m_wait; + } +private: + friend class d2::task_group; + friend class d2::task_group_base; + + bool continue_execution() const { + return m_wait.continue_execution(); + } + + wait_context m_wait; +}; + +struct execution_data { + task_group_context* context{}; + slot_id original_slot{}; + slot_id affinity_slot{}; +}; + +inline task_group_context* context(const execution_data& ed) { + return ed.context; +} + +inline slot_id original_slot(const execution_data& ed) { + return ed.original_slot; +} + +inline slot_id affinity_slot(const execution_data& ed) { + return ed.affinity_slot; +} + +inline slot_id execution_slot(const execution_data& ed) { + return r1::execution_slot(&ed); +} + +inline bool is_same_affinity(const execution_data& ed) { + return affinity_slot(ed) == no_slot || affinity_slot(ed) == execution_slot(ed); +} + +inline bool is_stolen(const execution_data& ed) { + return original_slot(ed) != execution_slot(ed); +} + +inline void spawn(task& t, task_group_context& ctx) { + call_itt_task_notify(releasing, &t); + r1::spawn(t, ctx); +} + +inline void spawn(task& t, task_group_context& ctx, slot_id id) { + call_itt_task_notify(releasing, &t); + r1::spawn(t, ctx, id); +} + +inline void execute_and_wait(task& t, task_group_context& t_ctx, wait_context& wait_ctx, task_group_context& w_ctx) { + r1::execute_and_wait(t, t_ctx, wait_ctx, w_ctx); + call_itt_task_notify(acquired, &wait_ctx); + call_itt_task_notify(destroy, &wait_ctx); +} + +inline void wait(wait_context& wait_ctx, task_group_context& ctx) { + r1::wait(wait_ctx, ctx); + call_itt_task_notify(acquired, &wait_ctx); + call_itt_task_notify(destroy, &wait_ctx); +} + +using r1::current_task_ptr; +using r1::current_context; + +class task_traits { + std::uint64_t m_version_and_traits{}; + friend struct r1::task_accessor; +}; + +//! Alignment for a task object +__TBB_GLOBAL_VAR constexpr std::size_t task_alignment = 64; + +//! Base class for user-defined tasks. +/** @ingroup task_scheduling */ +class alignas(task_alignment) task : public task_traits { +protected: + virtual ~task() = default; + +public: + virtual task* execute(execution_data&) = 0; + virtual task* cancel(execution_data&) = 0; + +private: + std::uint64_t m_reserved[6]{}; + friend struct r1::task_accessor; +}; +static_assert(sizeof(task) == task_alignment, "task size is broken"); + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif /* __TBB__task_H */ diff --git a/src/3rd party/oneapi/tbb/detail/_task_handle.h b/src/3rd party/oneapi/tbb/detail/_task_handle.h new file mode 100644 index 0000000000..9d76e65804 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_task_handle.h @@ -0,0 +1,699 @@ +/* + Copyright (c) 2020-2025 Intel Corporation + Copyright (c) 2025 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + + +#ifndef __TBB_task_handle_H +#define __TBB_task_handle_H + +#include "_config.h" +#include "_task.h" +#include "_small_object_pool.h" +#include "_utils.h" +#include + +namespace tbb { +namespace detail { + +namespace d1 { class task_group_context; class wait_context; struct execution_data; } +namespace d2 { + +class task_handle; + +enum task_group_status { + not_complete, + complete, + canceled +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + , task_complete +#endif +}; + +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + +class task_handle_task; +class task_dynamic_state; + +struct notify_list_node { + // First element is a pointer to a bypassed task + // Second element is true if bypass is allowed + using notify_result_type = std::pair; + + notify_list_node* next_node = nullptr; + + virtual notify_result_type notify_on_completion() = 0; + virtual notify_result_type notify_on_cancellation() = 0; + virtual ~notify_list_node() = default; +}; + +struct notify_successor_node : notify_list_node { + task_dynamic_state* successor_state = nullptr; + d1::small_object_allocator allocator; + + notify_successor_node(task_dynamic_state* state, d1::small_object_allocator& alloc) + : successor_state(state), allocator(alloc) {} + + notify_result_type notify_common(); + + notify_result_type notify_on_completion() override { + return notify_common(); + } + + notify_result_type notify_on_cancellation() override { + return notify_common(); + } +}; + +struct notify_waiter_node : notify_list_node { + d1::wait_context task_wait_context; + bool was_canceled; + + notify_waiter_node() : task_wait_context(1), was_canceled(false) {} + + notify_result_type notify_common() { + task_wait_context.release(); + + // Bypassing from list notification is not allowed if there are waiters in the list + return {nullptr, false}; + } + + virtual notify_result_type notify_on_completion() override { + return notify_common(); + } + + virtual notify_result_type notify_on_cancellation() override { + was_canceled = true; + return notify_common(); + } +}; + +class task_dynamic_state { +public: + task_dynamic_state(task_handle_task* task, d1::small_object_allocator& alloc) + : m_task(task) + , m_notify_list_head(nullptr) + , m_new_completion_point(nullptr) + , m_num_dependencies(0) + , m_num_references(1) // reserves a task co-ownership for dynamic state + , m_allocator(alloc) + {} + + void reserve() { ++m_num_references; } + + void release() { + if (--m_num_references == 0) { + task_dynamic_state* new_completion_point = m_new_completion_point.load(std::memory_order_relaxed); + // There was a new completion point assigned to the current one by transferring the completion + // Need to unregister the current dynamic state as a co-owner + if (new_completion_point) new_completion_point->release(); + m_allocator.delete_object(this); + } + } + + void register_dependency() { + if (m_num_dependencies++ == 0) { + // Register an additional dependency for a task_handle owning the current task + ++m_num_dependencies; + } + } + + // Returns true if the released dependency was the last remaining one; false otherwise + bool release_dependency() { + auto updated_dependency_counter = --m_num_dependencies; + return updated_dependency_counter == 0; + } + + bool has_dependencies() const { + return m_num_dependencies.load(std::memory_order_acquire) != 0; + } + + task_handle_task* complete_and_try_get_successor(); + task_handle_task* cancel_and_try_get_successor(); + + void add_successor(task_handle& successor); + task_group_status wait_for_completion(d1::task_group_context&); + task_group_status run_self_and_wait_for_completion(d1::task_group_context&); + void add_notify_node(notify_list_node* new_notify_node, notify_list_node* current_notify_list_head); + void add_notify_list(notify_list_node* notify_list); + + using notify_list_state_flag = std::uintptr_t; + static constexpr notify_list_state_flag COMPLETED_FLAG = ~std::uintptr_t(0); + static constexpr notify_list_state_flag TRANSFERRED_FLAG = ~std::uintptr_t(1); + static constexpr notify_list_state_flag CANCELED_FLAG = ~std::uintptr_t(2); + + static bool represents_completed_task(notify_list_node* list_head) { + return list_head == reinterpret_cast(COMPLETED_FLAG); + } + + static bool represents_canceled_task(notify_list_node* list_head) { + return list_head == reinterpret_cast(CANCELED_FLAG); + } + + static bool represents_transferred_completion(notify_list_node* list_head) { + return list_head == reinterpret_cast(TRANSFERRED_FLAG); + } + + task_handle_task* fetch_list_and_notify_all(notify_list_state_flag); + + notify_list_node* fetch_notify_list(notify_list_state_flag new_list_state_flag) { + return m_notify_list_head.exchange(reinterpret_cast(new_list_state_flag)); + } + + void transfer_completion_to(task_dynamic_state* new_completion_point) { + __TBB_ASSERT(new_completion_point != nullptr, nullptr); + // Register current dynamic state as a co-owner of the new_completion_point + // to prevent it's early destruction + new_completion_point->reserve(); + m_new_completion_point.store(new_completion_point, std::memory_order_relaxed); + notify_list_node* notify_list = fetch_notify_list(TRANSFERRED_FLAG); + new_completion_point->add_notify_list(notify_list); + } + + task_group_status get_task_status() { + notify_list_node* current_list_head = m_notify_list_head.load(std::memory_order_acquire); + task_group_status status = task_group_status::not_complete; + + if (represents_transferred_completion(current_list_head)) { + task_dynamic_state* new_completion_point = m_new_completion_point.load(std::memory_order_relaxed); + __TBB_ASSERT(new_completion_point != nullptr, nullptr); + status = new_completion_point->get_task_status(); + } else if (represents_completed_task(current_list_head)) { + status = task_group_status::task_complete; + } else if (represents_canceled_task(current_list_head)) { + status = task_group_status::canceled; + } + + return status; + } + + task_handle_task* get_task() { return m_task; } + +private: + task_handle_task* m_task; + std::atomic m_notify_list_head; + std::atomic m_new_completion_point; + std::atomic m_num_dependencies; + std::atomic m_num_references; + d1::small_object_allocator m_allocator; +}; + +inline std::pair notify_successor_node::notify_common() { + task_handle_task* successor_task = nullptr; + if (successor_state->release_dependency()) { + successor_task = successor_state->get_task(); + allocator.delete_object(this); + } + + return {successor_task, true}; +} + +class dynamic_state_task : public d1::task { + std::atomic m_dynamic_state; +public: + dynamic_state_task() : m_dynamic_state(nullptr) {} + + ~dynamic_state_task() { + task_dynamic_state* current_state = m_dynamic_state.load(std::memory_order_relaxed); + if (current_state != nullptr) { + current_state->release(); + } + } + + // Returns the dynamic state associated with the task. If the state has not been initialized, initializes it. + task_dynamic_state* get_dynamic_state(); + + task_handle_task* complete_and_try_get_successor() { + task_handle_task* next_task = nullptr; + + task_dynamic_state* current_state = m_dynamic_state.load(std::memory_order_relaxed); + if (current_state != nullptr) { + next_task = current_state->complete_and_try_get_successor(); + } + return next_task; + } + + task_handle_task* cancel_and_try_get_successor() { + task_handle_task* next_task = nullptr; + + task_dynamic_state* current_state = m_dynamic_state.load(std::memory_order_relaxed); + if (current_state != nullptr) { + next_task = current_state->cancel_and_try_get_successor(); + } + return next_task; + } + + // Returns true if the released dependency was the last remaining one; false otherwise + bool release_dependency() { + task_dynamic_state* current_state = m_dynamic_state.load(std::memory_order_relaxed); + __TBB_ASSERT(current_state != nullptr && current_state->has_dependencies(), + "release_dependency was called for task without dependencies"); + return current_state->release_dependency(); + } + + bool has_dependencies() const { + task_dynamic_state* current_state = m_dynamic_state.load(std::memory_order_relaxed); + return current_state ? current_state->has_dependencies() : false; + } + + void transfer_completion_to(task_handle& receiving_task); +}; +#endif // __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + +class task_handle_task +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + : public dynamic_state_task +#else + : public d1::task +#endif +{ + // Pointer to the instantiation of destroy_function_task with the concrete derived type, + // used for correct destruction and deallocation of the task + using destroy_func_type = void (*)(task_handle_task*, d1::small_object_allocator&, const d1::execution_data*); + + // Reuses the first std::uint64_t field (previously m_version_and_traits) to maintain backward compatibility + // The type of the first field remains std::uint64_t to preserve alignment and offset of subsequent member variables. + static_assert(sizeof(destroy_func_type) <= sizeof(std::uint64_t), "Cannot fit destroy pointer into std::uint64_t"); + std::uint64_t m_destroy_func; + + d1::wait_tree_vertex_interface* m_wait_tree_vertex; + d1::task_group_context& m_ctx; + d1::small_object_allocator m_allocator; +public: + void destroy(const d1::execution_data* ed = nullptr) { + destroy_func_type destroy_func = reinterpret_cast(m_destroy_func); + if (destroy_func != nullptr) { + // If the destroy function is set for the current instantiation - use it + (*destroy_func)(this, m_allocator, ed); + } else { + // Otherwise, the object was compiled with the old version of the library + // Destroy the object and let the memory leak since the derived type is unknown + // and the object cannot be deallocated properly + this->~task_handle_task(); + } + } + + task_handle_task(d1::wait_tree_vertex_interface* vertex, d1::task_group_context& ctx, + d1::small_object_allocator& alloc, destroy_func_type destroy_func) + : m_destroy_func(reinterpret_cast(destroy_func)) + , m_wait_tree_vertex(vertex) + , m_ctx(ctx) + , m_allocator(alloc) + { + m_wait_tree_vertex->reserve(); + } + + ~task_handle_task() override { + m_wait_tree_vertex->release(); + } + + d1::task_group_context& ctx() const { return m_ctx; } +}; + +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS +inline task_dynamic_state* dynamic_state_task::get_dynamic_state() { +#if __TBB_USE_OPTIONAL_RTTI + __TBB_ASSERT(dynamic_cast(this) != nullptr, "get_dynamic_state was called for a stack task"); +#endif + task_dynamic_state* current_state = m_dynamic_state.load(std::memory_order_acquire); + + if (current_state == nullptr) { + d1::small_object_allocator alloc; + + task_dynamic_state* new_state = alloc.new_object(static_cast(this), alloc); + + if (m_dynamic_state.compare_exchange_strong(current_state, new_state)) { + current_state = new_state; + } else { + // CAS failed, current_state points to the dynamic state created by another thread + alloc.delete_object(new_state); + } + } + + __TBB_ASSERT(current_state != nullptr, "Failed to create dynamic state"); + return current_state; +} +#endif + +class task_handle { + struct task_handle_task_deleter { + void operator()(task_handle_task* p){ p->destroy(); } + }; + using handle_impl_t = std::unique_ptr; + + handle_impl_t m_handle = {nullptr}; +public: + task_handle() = default; + task_handle(task_handle&&) = default; + task_handle& operator=(task_handle&&) = default; + + explicit operator bool() const noexcept { return static_cast(m_handle); } + + friend bool operator==(task_handle const& th, std::nullptr_t) noexcept; + friend bool operator==(std::nullptr_t, task_handle const& th) noexcept; + + friend bool operator!=(task_handle const& th, std::nullptr_t) noexcept; + friend bool operator!=(std::nullptr_t, task_handle const& th) noexcept; + +private: + friend struct task_handle_accessor; +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + friend class task_completion_handle; +#endif + + task_handle(task_handle_task* t) : m_handle {t}{} +}; + +struct task_handle_accessor { + static task_handle construct(task_handle_task* t) { return {t}; } + + static task_handle_task* release(task_handle& th) { + return th.m_handle.release(); + } + + static d1::task_group_context& ctx_of(task_handle& th) { + __TBB_ASSERT(th.m_handle, "ctx_of does not expect empty task_handle."); + return th.m_handle->ctx(); + } + +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + static task_dynamic_state* get_task_dynamic_state(task_handle& th) { + return th.m_handle->get_dynamic_state(); + } +#endif +}; + +inline bool operator==(task_handle const& th, std::nullptr_t) noexcept { + return th.m_handle == nullptr; +} +inline bool operator==(std::nullptr_t, task_handle const& th) noexcept { + return th.m_handle == nullptr; +} + +inline bool operator!=(task_handle const& th, std::nullptr_t) noexcept { + return th.m_handle != nullptr; +} + +inline bool operator!=(std::nullptr_t, task_handle const& th) noexcept { + return th.m_handle != nullptr; +} + +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS +inline void task_dynamic_state::add_notify_node(notify_list_node* new_notify_node, + notify_list_node* current_notify_list_head) +{ + __TBB_ASSERT(new_notify_node != nullptr, nullptr); + + new_notify_node->next_node = current_notify_list_head; + + while (!m_notify_list_head.compare_exchange_strong(current_notify_list_head, new_notify_node)) { + // Other thread updated the head of the list + + if (represents_completed_task(current_notify_list_head)) { + // Current task has completed while we tried to insert the node to the list + new_notify_node->notify_on_completion(); + break; + } else if (represents_canceled_task(current_notify_list_head)) { + // Current task has canceled while we tried to insert the node to the list + new_notify_node->notify_on_cancellation(); + break; + } else if (represents_transferred_completion(current_notify_list_head)) { + // Redirect notify_node to the task received the completion + task_dynamic_state* new_completion_point = m_new_completion_point.load(std::memory_order_relaxed); + __TBB_ASSERT(new_completion_point, "notify list is marked as transferred, but new dynamic state is not set"); + new_completion_point->add_notify_node(new_notify_node, new_completion_point->m_notify_list_head.load(std::memory_order_acquire)); + break; + } + + new_notify_node->next_node = current_notify_list_head; + } +} + + +inline void task_dynamic_state::add_successor(task_handle& successor) { + notify_list_node* current_notify_list_head = m_notify_list_head.load(std::memory_order_acquire); + + if (!represents_completed_task(current_notify_list_head) && !represents_canceled_task(current_notify_list_head)) { + if (represents_transferred_completion(current_notify_list_head)) { + // Redirect successor to the task received the completion + task_dynamic_state* new_completion_point = m_new_completion_point.load(std::memory_order_relaxed); + __TBB_ASSERT(new_completion_point, "notify list is marked as transferred, but new dynamic state is not set"); + new_completion_point->add_successor(successor); + } else { + task_dynamic_state* successor_state = task_handle_accessor::get_task_dynamic_state(successor); + successor_state->register_dependency(); + + d1::small_object_allocator alloc; + notify_successor_node* new_successor_node = alloc.new_object(successor_state, alloc); + add_notify_node(new_successor_node, current_notify_list_head); + } + } +} + +inline task_group_status task_dynamic_state::wait_for_completion(d1::task_group_context& ctx) { + notify_list_node* current_notify_list_head = m_notify_list_head.load(std::memory_order_acquire); + task_group_status status = task_group_status::not_complete; + + if (represents_completed_task(current_notify_list_head)) { + status = task_group_status::task_complete; + } else if (represents_canceled_task(current_notify_list_head)) { + status = task_group_status::canceled; + } else if (represents_transferred_completion(current_notify_list_head)) { + // Redirect waiter to the task received the completion + task_dynamic_state* new_completion_point = m_new_completion_point.load(std::memory_order_relaxed); + __TBB_ASSERT(new_completion_point, "notify list is marked as transferred, but new dynamic state is not set"); + status = new_completion_point->wait_for_completion(ctx); + } else { + notify_waiter_node waiter_node; + add_notify_node(&waiter_node, current_notify_list_head); + d1::wait(waiter_node.task_wait_context, ctx); + status = waiter_node.was_canceled ? task_group_status::canceled : task_group_status::task_complete; + } + + return status; +} + +inline task_group_status task_dynamic_state::run_self_and_wait_for_completion(d1::task_group_context& ctx) { + __TBB_ASSERT(!has_dependencies(), nullptr); + notify_list_node* current_notify_list_head = m_notify_list_head.load(std::memory_order_acquire); + + __TBB_ASSERT(!represents_completed_task(current_notify_list_head), "non-submitted task cannot be completed"); + __TBB_ASSERT(!represents_canceled_task(current_notify_list_head), "non-submitted task cannot be canceled"); + __TBB_ASSERT(!represents_transferred_completion(current_notify_list_head), "non-submitted task completion cannot be transferred"); + + notify_waiter_node waiter_node; + add_notify_node(&waiter_node, current_notify_list_head); + d1::execute_and_wait(*get_task(), ctx, waiter_node.task_wait_context, ctx); + return waiter_node.was_canceled ? task_group_status::canceled : task_group_status::task_complete; +} + +inline void task_dynamic_state::add_notify_list(notify_list_node* notify_list) { + if (notify_list == nullptr) return; + + notify_list_node* last_node = notify_list; + + while (last_node->next_node != nullptr) { + last_node = last_node->next_node; + } + + notify_list_node* current_notify_list_head = m_notify_list_head.load(std::memory_order_acquire); + last_node->next_node = current_notify_list_head; + + while (!m_notify_list_head.compare_exchange_strong(current_notify_list_head, notify_list)) { + __TBB_ASSERT(!represents_completed_task(current_notify_list_head) && + !represents_canceled_task(current_notify_list_head) && + !represents_transferred_completion(current_notify_list_head), + "Task receiving the completion was executed or completed"); + // Other thread updated the head of the list + last_node->next_node = current_notify_list_head; + } +} + +// Notifies +inline task_handle_task* task_dynamic_state::fetch_list_and_notify_all(notify_list_state_flag state_flag) { + __TBB_ASSERT(state_flag == COMPLETED_FLAG || state_flag == CANCELED_FLAG, "Unexpected state_flag"); + notify_list_node* node = fetch_notify_list(state_flag); + task_handle_task* next_task = nullptr; + bool bypass_allowed = true; + + while (node != nullptr) { + notify_list_node* next_node = node->next_node; + + // Don't dereference node after the notification! + notify_list_node::notify_result_type result = state_flag == COMPLETED_FLAG ? + node->notify_on_completion() : + node->notify_on_cancellation(); + if (!result.second) bypass_allowed = false; + task_handle_task* successor_task = result.first; + + if (next_task == nullptr) { + next_task = successor_task; + } else if (successor_task != nullptr) { + d1::spawn(*successor_task, successor_task->ctx()); + } + + node = next_node; + } + + if (next_task && !bypass_allowed) { + d1::spawn(*next_task, next_task->ctx()); + next_task = nullptr; + } + return next_task; +} + +inline task_handle_task* task_dynamic_state::complete_and_try_get_successor() { + task_handle_task* next_task = nullptr; + notify_list_node* node = m_notify_list_head.load(std::memory_order_acquire); + + // Doing a single check is enough since the this function is called after the task body and + // the state of the list cannot change to transferred + if (!represents_transferred_completion(node)) { + next_task = fetch_list_and_notify_all(COMPLETED_FLAG); + } + return next_task; +} + +inline task_handle_task* task_dynamic_state::cancel_and_try_get_successor() { + __TBB_ASSERT(!represents_transferred_completion(m_notify_list_head.load(std::memory_order_relaxed)), + "canceled task completion cannot be transferred"); + return fetch_list_and_notify_all(CANCELED_FLAG); +} + +inline void dynamic_state_task::transfer_completion_to(task_handle& receiving_task) { + __TBB_ASSERT(receiving_task, nullptr); + task_dynamic_state* current_state = m_dynamic_state.load(std::memory_order_relaxed); + + // If dynamic state was not created for currently executing task, + // it cannot have successors or associated completion handles + if (current_state != nullptr) { + current_state->transfer_completion_to(task_handle_accessor::get_task_dynamic_state(receiving_task)); + } +} + +class task_completion_handle { +public: + task_completion_handle() : m_task_state(nullptr) {} + + task_completion_handle(const task_completion_handle& other) + : m_task_state(other.m_task_state) + { + // Register one more co-owner of the dynamic state + if (m_task_state) m_task_state->reserve(); + } + task_completion_handle(task_completion_handle&& other) + : m_task_state(other.m_task_state) + { + other.m_task_state = nullptr; + } + + task_completion_handle(const task_handle& th) + : m_task_state(nullptr) + { + __TBB_ASSERT(th, "Construction of task_completion_handle from an empty task_handle"); + m_task_state = th.m_handle->get_dynamic_state(); + // Register one more co-owner of the dynamic state + m_task_state->reserve(); + } + + ~task_completion_handle() { + if (m_task_state) m_task_state->release(); + } + + task_completion_handle& operator=(const task_completion_handle& other) { + if (m_task_state != other.m_task_state) { + // Release co-ownership on the previously tracked dynamic state + if (m_task_state) m_task_state->release(); + + m_task_state = other.m_task_state; + + // Register new co-owner of the new dynamic state + if (m_task_state) m_task_state->reserve(); + } + return *this; + } + + task_completion_handle& operator=(task_completion_handle&& other) { + if (this != &other) { + // Release co-ownership on the previously tracked dynamic state + if (m_task_state) m_task_state->release(); + + m_task_state = other.m_task_state; + other.m_task_state = nullptr; + } + return *this; + } + + task_completion_handle& operator=(const task_handle& th) { + __TBB_ASSERT(th, "Assignment of task_completion_state from an empty task_handle"); + task_dynamic_state* th_state = th.m_handle->get_dynamic_state(); + __TBB_ASSERT(th_state != nullptr, "No state in the non-empty task_handle"); + if (m_task_state != th_state) { + // Release co-ownership on the previously tracked dynamic state + if (m_task_state) m_task_state->release(); + + m_task_state = th_state; + + // Reserve co-ownership on the new dynamic state + m_task_state->reserve(); + } + return *this; + } + + explicit operator bool() const noexcept { return m_task_state != nullptr; } +private: + friend bool operator==(const task_completion_handle& t, std::nullptr_t) noexcept { + return t.m_task_state == nullptr; + } + + friend bool operator==(const task_completion_handle& lhs, const task_completion_handle& rhs) noexcept { + return lhs.m_task_state == rhs.m_task_state; + } + +#if !__TBB_CPP20_COMPARISONS_PRESENT + friend bool operator==(std::nullptr_t, const task_completion_handle& t) noexcept { + return t == nullptr; + } + + friend bool operator!=(const task_completion_handle& t, std::nullptr_t) noexcept { + return !(t == nullptr); + } + + friend bool operator!=(std::nullptr_t, const task_completion_handle& t) noexcept { + return !(t == nullptr); + } + + friend bool operator!=(const task_completion_handle& lhs, const task_completion_handle& rhs) noexcept { + return !(lhs == rhs); + } +#endif // !__TBB_CPP20_COMPARISONS_PRESENT + + friend struct task_completion_handle_accessor; + + task_dynamic_state* m_task_state; +}; + +struct task_completion_handle_accessor { + static task_dynamic_state* get_task_dynamic_state(task_completion_handle& tracker) { + return tracker.m_task_state; + } +}; +#endif + +} // namespace d2 +} // namespace detail +} // namespace tbb + +#endif /* __TBB_task_handle_H */ diff --git a/src/3rd party/oneapi/tbb/detail/_template_helpers.h b/src/3rd party/oneapi/tbb/detail/_template_helpers.h new file mode 100644 index 0000000000..a20c5af5c3 --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_template_helpers.h @@ -0,0 +1,407 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__template_helpers_H +#define __TBB_detail__template_helpers_H + +#include "_utils.h" +#include "_config.h" + +#include +#include +#include +#include +#include +#include + +namespace tbb { +namespace detail { +inline namespace d0 { + +// An internal implementation of void_t, which can be used in SFINAE contexts +template +struct void_impl { + using type = void; +}; // struct void_impl + +template +using void_t = typename void_impl::type; + +// Generic SFINAE helper for expression checks, based on the idea demonstrated in ISO C++ paper n4502 +template class... Checks> +struct supports_impl { + using type = std::false_type; +}; + +template class... Checks> +struct supports_impl...>, Checks...> { + using type = std::true_type; +}; + +template class... Checks> +using supports = typename supports_impl::type; + +//! A template to select either 32-bit or 64-bit constant as compile time, depending on machine word size. +template +struct select_size_t_constant { + // Explicit cast is needed to avoid compiler warnings about possible truncation. + // The value of the right size, which is selected by ?:, is anyway not truncated or promoted. + static const std::size_t value = static_cast((sizeof(std::size_t)==sizeof(u)) ? u : ull); +}; + +// TODO: do we really need it? +//! Cast between unrelated pointer types. +/** This method should be used sparingly as a last resort for dealing with + situations that inherently break strict ISO C++ aliasing rules. */ +// T is a pointer type because it will be explicitly provided by the programmer as a template argument; +// U is a referent type to enable the compiler to check that "ptr" is a pointer, deducing U in the process. +template +inline T punned_cast( U* ptr ) { + std::uintptr_t x = reinterpret_cast(ptr); + return reinterpret_cast(x); +} + +template +struct padded_base : T { + char pad[S - R]; +}; +template struct padded_base : T {}; + +//! Pads type T to fill out to a multiple of cache line size. +template +struct padded : padded_base {}; + +#if __TBB_CPP14_INTEGER_SEQUENCE_PRESENT + +using std::index_sequence; +using std::make_index_sequence; + +#else + +template class index_sequence {}; + +template +struct make_index_sequence_impl : make_index_sequence_impl < N - 1, N - 1, S... > {}; + +template +struct make_index_sequence_impl <0, S...> { + using type = index_sequence; +}; + +template +using make_index_sequence = typename make_index_sequence_impl::type; + +#endif /* __TBB_CPP14_INTEGER_SEQUENCE_PRESENT */ + +//! Attach an index to a type to use it with an index sequence +template +using indexed_t = T; + +#if __TBB_CPP17_LOGICAL_OPERATIONS_PRESENT +using std::conjunction; +using std::disjunction; +#else // __TBB_CPP17_LOGICAL_OPERATIONS_PRESENT + +template +struct conjunction : std::true_type {}; + +template +struct conjunction + : std::conditional, First>::type {}; + +template +struct conjunction : T {}; + +template +struct disjunction : std::false_type {}; + +template +struct disjunction + : std::conditional>::type {}; + +template +struct disjunction : T {}; + +#endif // __TBB_CPP17_LOGICAL_OPERATIONS_PRESENT + +template +using iterator_value_t = typename std::iterator_traits::value_type; + +template +using iterator_key_t = typename std::remove_const::first_type>::type; + +template +using iterator_mapped_t = typename iterator_value_t::second_type; + +template +using iterator_alloc_pair_t = std::pair>::type, + iterator_mapped_t>; + +template using alloc_value_type = typename A::value_type; +template using alloc_ptr_t = typename std::allocator_traits::pointer; +template using has_allocate = decltype(std::declval&>() = std::declval().allocate(0)); +template using has_deallocate = decltype(std::declval().deallocate(std::declval>(), 0)); + +// alloc_value_type should be checked first, because it can be used in other checks +template +using is_allocator = supports; + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +template +inline constexpr bool is_allocator_v = is_allocator::value; +#endif + +// Template class in which the "type" determines the type of the element number N in pack Args +template +struct pack_element { + using type = void; +}; + +template +struct pack_element { + using type = typename pack_element::type; +}; + +template +struct pack_element<0, T, Args...> { + using type = T; +}; + +template +using pack_element_t = typename pack_element::type; + +template +class raii_guard { +public: + static_assert( + std::is_nothrow_copy_constructible::value && + std::is_nothrow_move_constructible::value, + "Throwing an exception during the Func copy or move construction cause an unexpected behavior." + ); + + raii_guard( Func f ) noexcept : my_func(f), is_active(true) {} + + raii_guard( raii_guard&& g ) noexcept : my_func(std::move(g.my_func)), is_active(g.is_active) { + g.is_active = false; + } + + ~raii_guard() { + if (is_active) { + my_func(); + } + } + + void dismiss() { + is_active = false; + } +private: + Func my_func; + bool is_active; +}; // class raii_guard + +template +raii_guard make_raii_guard( Func f ) { + return raii_guard(f); +} + +template +struct try_call_proxy { + try_call_proxy( Body b ) : body(b) {} + + template + void on_exception( OnExceptionBody on_exception_body ) { + auto guard = make_raii_guard(on_exception_body); + body(); + guard.dismiss(); + } + + template + void on_completion(OnCompletionBody on_completion_body) { + auto guard = make_raii_guard(on_completion_body); + body(); + } + + Body body; +}; // struct try_call_proxy + +// Template helper function for API +// try_call(lambda1).on_exception(lambda2) +// Executes lambda1 and if it throws an exception - executes lambda2 +template +try_call_proxy try_call( Body b ) { + return try_call_proxy(b); +} + +#if __TBB_CPP17_IS_SWAPPABLE_PRESENT +using std::is_nothrow_swappable; +using std::is_swappable; +#else // __TBB_CPP17_IS_SWAPPABLE_PRESENT +namespace is_swappable_detail { +using std::swap; + +template +using has_swap = decltype(swap(std::declval(), std::declval())); + +#if _MSC_VER && _MSC_VER <= 1900 && !__INTEL_COMPILER +// Workaround for VS2015: it fails to instantiate noexcept(...) inside std::integral_constant. +template +struct noexcept_wrapper { + static const bool value = noexcept(swap(std::declval(), std::declval())); +}; +template +struct is_nothrow_swappable_impl : std::integral_constant::value> {}; +#else +template +struct is_nothrow_swappable_impl : std::integral_constant(), std::declval()))> {}; +#endif +} + +template +struct is_swappable : supports {}; + +template +struct is_nothrow_swappable + : conjunction, is_swappable_detail::is_nothrow_swappable_impl> {}; +#endif // __TBB_CPP17_IS_SWAPPABLE_PRESENT + +//! Allows to store a function parameter pack as a variable and later pass it to another function +template< typename... Types > +struct stored_pack; + +template<> +struct stored_pack<> +{ + using pack_type = stored_pack<>; + stored_pack() {} + + // Friend front-end functions + template< typename F, typename Pack > friend void call(F&& f, Pack&& p); + template< typename Ret, typename F, typename Pack > friend Ret call_and_return(F&& f, Pack&& p); + +protected: + // Ideally, ref-qualified non-static methods would be used, + // but that would greatly reduce the set of compilers where it works. + template< typename Ret, typename F, typename... Preceding > + static Ret call(F&& f, const pack_type& /*pack*/, Preceding&&... params) { + return std::forward(f)(std::forward(params)...); + } + template< typename Ret, typename F, typename... Preceding > + static Ret call(F&& f, pack_type&& /*pack*/, Preceding&&... params) { + return std::forward(f)(std::forward(params)...); + } +}; + +template< typename T, typename... Types > +struct stored_pack : stored_pack +{ + using pack_type = stored_pack; + using pack_remainder = stored_pack; + + // Since lifetime of original values is out of control, copies should be made. + // Thus references should be stripped away from the deduced type. + typename std::decay::type leftmost_value; + + // Here rvalue references act in the same way as forwarding references, + // as long as class template parameters were deduced via forwarding references. + stored_pack(T&& t, Types&&... types) + : pack_remainder(std::forward(types)...), leftmost_value(std::forward(t)) {} + + // Friend front-end functions + template< typename F, typename Pack > friend void call(F&& f, Pack&& p); + template< typename Ret, typename F, typename Pack > friend Ret call_and_return(F&& f, Pack&& p); + +protected: + template< typename Ret, typename F, typename... Preceding > + static Ret call(F&& f, pack_type& pack, Preceding&&... params) { + return pack_remainder::template call( + std::forward(f), static_cast(pack), + std::forward(params)... , pack.leftmost_value + ); + } + + template< typename Ret, typename F, typename... Preceding > + static Ret call(F&& f, pack_type&& pack, Preceding&&... params) { + return pack_remainder::template call( + std::forward(f), static_cast(pack), + std::forward(params)... , std::move(pack.leftmost_value) + ); + } +}; + +//! Calls the given function with arguments taken from a stored_pack +template< typename F, typename Pack > +void call(F&& f, Pack&& p) { + std::decay::type::template call(std::forward(f), std::forward(p)); +} + +template< typename Ret, typename F, typename Pack > +Ret call_and_return(F&& f, Pack&& p) { + return std::decay::type::template call(std::forward(f), std::forward(p)); +} + +template< typename... Types > +stored_pack save_pack(Types&&... types) { + return stored_pack(std::forward(types)...); +} + +// A structure with the value which is equal to Trait::value +// but can be used in the immediate context due to parameter T +template +struct dependent_bool : std::integral_constant {}; + +template +struct body_arg_detector; + +template +struct body_arg_detector { + using arg_type = Arg; +}; + +template +struct body_arg_detector { + using arg_type = Arg; +}; + +template +struct argument_detector; + +template +struct argument_detector { + using type = typename body_arg_detector::arg_type; +}; + +template +struct argument_detector { + using type = Arg; +}; + +// Detects the argument type of callable, works for callable with one argument. +template +using argument_type_of = typename argument_detector::type>::type; + +template +struct type_identity { + using type = T; +}; + +template +using type_identity_t = typename type_identity::type; + +} // inline namespace d0 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__template_helpers_H diff --git a/src/3rd party/oneapi/tbb/detail/_utils.h b/src/3rd party/oneapi/tbb/detail/_utils.h new file mode 100644 index 0000000000..922b508f0f --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_utils.h @@ -0,0 +1,449 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__utils_H +#define __TBB_detail__utils_H + +#include +#include +#include +#include +#include +#include + +#include "_config.h" +#include "_assert.h" +#include "_machine.h" + +namespace tbb { +namespace detail { +inline namespace d0 { + +//! Utility template function to prevent "unused" warnings by various compilers. +template void suppress_unused_warning(T&&...) {} + +//! Compile-time constant that is upper bound on cache line/sector size. +/** It should be used only in situations where having a compile-time upper + bound is more useful than a run-time exact answer. + @ingroup memory_allocation */ +__TBB_GLOBAL_VAR constexpr size_t max_nfs_size = 128; +__TBB_GLOBAL_VAR constexpr std::size_t max_nfs_size_exp = 7; +static_assert(1 << max_nfs_size_exp == max_nfs_size, "max_nfs_size_exp must be a log2(max_nfs_size)"); + +//! Class that implements exponential backoff. +class atomic_backoff { + //! Time delay, in units of "pause" instructions. + /** Should be equal to approximately the number of "pause" instructions + that take the same time as an context switch. Must be a power of two.*/ + static constexpr std::int32_t LOOPS_BEFORE_YIELD = 16; + std::int32_t count; + +public: + // In many cases, an object of this type is initialized eagerly on hot path, + // as in for(atomic_backoff b; ; b.pause()) { /*loop body*/ } + // For this reason, the construction cost must be very small! + atomic_backoff() : count(1) {} + // This constructor pauses immediately; do not use on hot paths! + atomic_backoff(bool) : count(1) { pause(); } + + //! No Copy + atomic_backoff(const atomic_backoff&) = delete; + atomic_backoff& operator=(const atomic_backoff&) = delete; + + //! Pause for a while. + void pause() { + if (count <= LOOPS_BEFORE_YIELD) { + machine_pause(count); + // Pause twice as long the next time. + count *= 2; + } else { + // Pause is so long that we might as well yield CPU to scheduler. + yield(); + } + } + + //! Pause for a few times and return false if saturated. + bool bounded_pause() { + machine_pause(count); + if (count < LOOPS_BEFORE_YIELD) { + // Pause twice as long the next time. + count *= 2; + return true; + } else { + return false; + } + } + + void reset() { + count = 1; + } +}; + +//! Spin WHILE the condition is true. +/** T and U should be comparable types. */ +template +T spin_wait_while(const std::atomic& location, C comp, std::memory_order order) { + atomic_backoff backoff; + T snapshot = location.load(order); + while (comp(snapshot)) { + backoff.pause(); + snapshot = location.load(order); + } + return snapshot; +} + +//! Spin WHILE the value of the variable is equal to a given value +/** T and U should be comparable types. */ +template +T spin_wait_while_eq(const std::atomic& location, const U value, std::memory_order order = std::memory_order_acquire) { + return spin_wait_while(location, [&value](T t) { return t == value; }, order); +} + +//! Spin UNTIL the value of the variable is equal to a given value +/** T and U should be comparable types. */ +template +T spin_wait_until_eq(const std::atomic& location, const U value, std::memory_order order = std::memory_order_acquire) { + return spin_wait_while(location, [&value](T t) { return t != value; }, order); +} + +//! Spin UNTIL the condition returns true or spinning time is up. +/** Returns what the passed functor returned last time it was invoked. */ +template +bool timed_spin_wait_until(Condition condition) { + // 32 pauses + 32 yields are meausered as balanced spin time before sleep. + bool finish = condition(); + for (int i = 1; !finish && i < 32; finish = condition(), i *= 2) { + machine_pause(i); + } + for (int i = 32; !finish && i < 64; finish = condition(), ++i) { + yield(); + } + return finish; +} + +template +T clamp(T value, T lower_bound, T upper_bound) { + __TBB_ASSERT(lower_bound <= upper_bound, "Incorrect bounds"); + return value > lower_bound ? (value > upper_bound ? upper_bound : value) : lower_bound; +} + +template +std::uintptr_t log2(T in) { + __TBB_ASSERT(in > 0, "The logarithm of a non-positive value is undefined."); + return machine_log2(in); +} + +template +T reverse_bits(T src) { + return machine_reverse_bits(src); +} + +template +T reverse_n_bits(T src, std::size_t n) { + __TBB_ASSERT(n != 0, "Reverse for 0 bits is undefined behavior."); + return reverse_bits(src) >> (number_of_bits() - n); +} + +//! Encodes/decodes multiple core type IDs into/from a single integer value using bitmask +struct multi_core_type_codec { + using core_type_id = int; + static constexpr core_type_id automatic = -1; + + static core_type_id encode(const std::vector& ids) { + if (ids.empty()) { + return automatic; + } + if (ids.size() == 1) { + return ids[0]; + } + + core_type_id result = core_type_id(encoding_format << bitmask_width); + + for (core_type_id id : ids) { + __TBB_ASSERT((0 <= id) && (id < static_cast(bitmask_width)), "Wrong core type id"); + result |= (1 << id); + } + + return result; + } + static std::vector decode(core_type_id core_type) { + if (!is_encoded(core_type)) { + return {core_type}; + } + + std::vector core_type_ids; + for (size_t bit_pos = 0; bit_pos < bitmask_width; ++bit_pos) { + if (core_type & (1 << bit_pos)) { + core_type_ids.push_back(static_cast(bit_pos)); + } + } + return core_type_ids; + } + static bool is_single(core_type_id id) { + return (id >> bitmask_width) == 0; + } + static bool is_encoded(core_type_id id) { + return (static_cast::type>(id) >> bitmask_width) == encoding_format; + } + static bool is_core_type(core_type_id id) { + return is_single(id) || is_encoded(id); + } + + // Lower bitmask_width bits encode IDs + static constexpr size_t bitmask_width = sizeof(core_type_id) * CHAR_BIT - 4; + + // Upper 4 bits: MSb=1 (makes result negative; real core type IDs are non-negative) + 3-bit format version + // (current: 0, max: 6; 1111 is excluded to avoid collision with plain negatives: -1, -2, ..., -268435456) + static constexpr size_t encoding_format = 0x8; +}; + +// A function to check if passed integer is a power of two +template +constexpr bool is_power_of_two( IntegerType arg ) { + static_assert(std::is_integral::value, + "An argument for is_power_of_two should be integral type"); + return arg && (0 == (arg & (arg - 1))); +} + +// A function to determine if passed integer is a power of two +// at least as big as another power of two, i.e. for strictly positive i and j, +// with j being a power of two, determines whether i==j< +constexpr bool is_power_of_two_at_least(ArgIntegerType arg, DivisorIntegerType divisor) { + // Divisor should be a power of two + static_assert(std::is_integral::value, + "An argument for is_power_of_two_at_least should be integral type"); + return 0 == (arg & (arg - divisor)); +} + +// A function to compute arg modulo divisor where divisor is a power of 2. +template +inline ArgIntegerType modulo_power_of_two(ArgIntegerType arg, DivisorIntegerType divisor) { + __TBB_ASSERT( is_power_of_two(divisor), "Divisor should be a power of two" ); + return arg & (divisor - 1); +} + +//! A function to check if passed in pointer is aligned on a specific border +template +constexpr bool is_aligned(T* pointer, std::uintptr_t alignment) { + return 0 == (reinterpret_cast(pointer) & (alignment - 1)); +} + +#if TBB_USE_ASSERT +__TBB_GLOBAL_VAR void* const poisoned_ptr = reinterpret_cast(-1); + +//! Set p to invalid pointer value. +template +inline void poison_pointer( T* &p ) { p = reinterpret_cast(poisoned_ptr); } + +template +inline void poison_pointer(std::atomic& p) { p.store(reinterpret_cast(poisoned_ptr), std::memory_order_relaxed); } + +/** Expected to be used in assertions only, thus no empty form is defined. **/ +template +inline bool is_poisoned( T* p ) { return p == reinterpret_cast(poisoned_ptr); } + +template +inline bool is_poisoned(const std::atomic& p) { return is_poisoned(p.load(std::memory_order_relaxed)); } +#else +template +inline void poison_pointer(T&) {/*do nothing*/} +#endif /* !TBB_USE_ASSERT */ + +template +bool assert_pointer_valid(T* p, const char* comment = nullptr) { + suppress_unused_warning(p, comment); + __TBB_ASSERT(p != nullptr, comment); + __TBB_ASSERT(!is_poisoned(p), comment); +#if !(_MSC_VER && _MSC_VER <= 1900 && !__INTEL_COMPILER) + __TBB_ASSERT(is_aligned(p, alignment == 0 ? alignof(T) : alignment), comment); +#endif + // Returns something to simplify assert_pointers_valid implementation. + return true; +} + +template +void assert_pointers_valid(Args*... p) { + // suppress_unused_warning is used as an evaluation context for the variadic pack. + suppress_unused_warning(assert_pointer_valid(p)...); +} + +//! Base class for types that should not be assigned. +class no_assign { +public: + void operator=(const no_assign&) = delete; + no_assign(const no_assign&) = default; + no_assign() = default; +}; + +//! Base class for types that should not be copied or assigned. +class no_copy: no_assign { +public: + no_copy(const no_copy&) = delete; + no_copy() = default; +}; + +template +void swap_atomics_relaxed(std::atomic& lhs, std::atomic& rhs){ + T tmp = lhs.load(std::memory_order_relaxed); + lhs.store(rhs.load(std::memory_order_relaxed), std::memory_order_relaxed); + rhs.store(tmp, std::memory_order_relaxed); +} + +//! One-time initialization states +enum class do_once_state { + uninitialized = 0, ///< No execution attempts have been undertaken yet + pending, ///< A thread is executing associated do-once routine + executed, ///< Do-once routine has been executed + initialized = executed ///< Convenience alias +}; + +//! One-time initialization function +/** /param initializer Pointer to function without arguments + The variant that returns bool is used for cases when initialization can fail + and it is OK to continue execution, but the state should be reset so that + the initialization attempt was repeated the next time. + /param state Shared state associated with initializer that specifies its + initialization state. Must be initially set to #uninitialized value + (e.g. by means of default static zero initialization). **/ +template +void atomic_do_once( const F& initializer, std::atomic& state ) { + // The loop in the implementation is necessary to avoid race when thread T2 + // that arrived in the middle of initialization attempt by another thread T1 + // has just made initialization possible. + // In such a case T2 has to rely on T1 to initialize, but T1 may already be past + // the point where it can recognize the changed conditions. + do_once_state expected_state; + while ( state.load( std::memory_order_acquire ) != do_once_state::executed ) { + if( state.load( std::memory_order_relaxed ) == do_once_state::uninitialized ) { + expected_state = do_once_state::uninitialized; +#if defined(__INTEL_COMPILER) && __INTEL_COMPILER <= 1910 + using enum_type = typename std::underlying_type::type; + if( ((std::atomic&)state).compare_exchange_strong( (enum_type&)expected_state, (enum_type)do_once_state::pending ) ) { +#else + if( state.compare_exchange_strong( expected_state, do_once_state::pending ) ) { +#endif + run_initializer( initializer, state ); + break; + } + } + spin_wait_while_eq( state, do_once_state::pending ); + } +} + +// Run the initializer which can not fail +template +void run_initializer(const Functor& f, std::atomic& state ) { + f(); + state.store(do_once_state::executed, std::memory_order_release); +} + +#if __TBB_CPP20_CONCEPTS_PRESENT +template +concept boolean_testable_impl = std::convertible_to; + +template +concept boolean_testable = boolean_testable_impl && requires( T&& t ) { + { !std::forward(t) } -> boolean_testable_impl; + }; + +#if __TBB_CPP20_COMPARISONS_PRESENT +struct synthesized_three_way_comparator { + template + auto operator()( const T1& lhs, const T2& rhs ) const + requires requires { + { lhs < rhs } -> boolean_testable; + { rhs < lhs } -> boolean_testable; + } + { + if constexpr (std::three_way_comparable_with) { + return lhs <=> rhs; + } else { + if (lhs < rhs) { + return std::weak_ordering::less; + } + if (rhs < lhs) { + return std::weak_ordering::greater; + } + return std::weak_ordering::equivalent; + } + } +}; // struct synthesized_three_way_comparator + +template +using synthesized_three_way_result = decltype(synthesized_three_way_comparator{}(std::declval(), + std::declval())); + +#endif // __TBB_CPP20_COMPARISONS_PRESENT + +// Check if the type T is implicitly OR explicitly convertible to U +template +concept relaxed_convertible_to = std::constructible_from; + +template +concept adaptive_same_as = +#if __TBB_STRICT_CONSTRAINTS + std::same_as; +#else + std::convertible_to; +#endif +#endif // __TBB_CPP20_CONCEPTS_PRESENT + +template +auto invoke(F&& f, Args&&... args) +#if __TBB_CPP17_INVOKE_PRESENT + noexcept(std::is_nothrow_invocable_v) + -> std::invoke_result_t +{ + return std::invoke(std::forward(f), std::forward(args)...); +} +#else // __TBB_CPP17_INVOKE_PRESENT + noexcept(noexcept(std::forward(f)(std::forward(args)...))) + -> decltype(std::forward(f)(std::forward(args)...)) +{ + return std::forward(f)(std::forward(args)...); +} +#endif // __TBB_CPP17_INVOKE_PRESENT + +} // namespace d0 + +namespace d1 { + +class delegate_base { +public: + virtual bool operator()() const = 0; + virtual ~delegate_base() {} +}; + +template +class delegated_function : public delegate_base { +public: + delegated_function(FuncType& f) : my_func(f) {} + + bool operator()() const override { + return my_func(); + } + +private: + FuncType &my_func; +}; +} // namespace d1 + +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__utils_H diff --git a/src/3rd party/oneapi/tbb/detail/_waitable_atomic.h b/src/3rd party/oneapi/tbb/detail/_waitable_atomic.h new file mode 100644 index 0000000000..1b18d11e5a --- /dev/null +++ b/src/3rd party/oneapi/tbb/detail/_waitable_atomic.h @@ -0,0 +1,90 @@ +/* + Copyright (c) 2021-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_detail__address_waiters_H +#define __TBB_detail__address_waiters_H + +#include "_utils.h" + +namespace tbb { +namespace detail { + +namespace r1 { +TBB_EXPORT void __TBB_EXPORTED_FUNC wait_on_address(void* address, d1::delegate_base& wakeup_condition, std::uintptr_t context); +TBB_EXPORT void __TBB_EXPORTED_FUNC notify_by_address(void* address, std::uintptr_t context); +TBB_EXPORT void __TBB_EXPORTED_FUNC notify_by_address_one(void* address); +TBB_EXPORT void __TBB_EXPORTED_FUNC notify_by_address_all(void* address); +} // namespace r1 + +namespace d1 { + +template +void adaptive_wait_on_address(void* address, Predicate wakeup_condition, std::uintptr_t context) { + if (!timed_spin_wait_until(wakeup_condition)) { + d1::delegated_function pred(wakeup_condition); + r1::wait_on_address(address, pred, context); + } +} + +template +class waitable_atomic { +public: + waitable_atomic() = default; + + explicit waitable_atomic(T value) : my_atomic(value) {} + + waitable_atomic(const waitable_atomic&) = delete; + waitable_atomic& operator=(const waitable_atomic&) = delete; + + T load(std::memory_order order) const noexcept { + return my_atomic.load(order); + } + + T exchange(T desired) noexcept { + return my_atomic.exchange(desired); + } + + void wait(T old, std::uintptr_t context, std::memory_order order) { + auto wakeup_condition = [&] { return my_atomic.load(order) != old; }; + if (!timed_spin_wait_until(wakeup_condition)) { + // We need to use while here, because notify_all() will wake up all threads + // But predicate for them might be false + d1::delegated_function pred(wakeup_condition); + do { + r1::wait_on_address(this, pred, context); + } while (!wakeup_condition()); + } + } + + void notify_one_relaxed() { + r1::notify_by_address_one(this); + } + + // TODO: consider adding following interfaces: + // store(desired, memory_order) + // notify_all_relaxed() + // wait_until(T, std::uintptr_t, std::memory_order) + // notify_relaxed(std::uintptr_t context) + +private: + std::atomic my_atomic{}; +}; + +} // namespace d1 +} // namespace detail +} // namespace tbb + +#endif // __TBB_detail__address_waiters_H diff --git a/src/3rd party/oneapi/tbb/enumerable_thread_specific.h b/src/3rd party/oneapi/tbb/enumerable_thread_specific.h new file mode 100644 index 0000000000..fabc7d5fcf --- /dev/null +++ b/src/3rd party/oneapi/tbb/enumerable_thread_specific.h @@ -0,0 +1,1126 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_enumerable_thread_specific_H +#define __TBB_enumerable_thread_specific_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_assert.h" +#include "detail/_template_helpers.h" +#include "detail/_aligned_space.h" + +#include "concurrent_vector.h" +#include "tbb_allocator.h" +#include "cache_aligned_allocator.h" +#include "profiling.h" + +#include +#include +#include // memcpy +#include // std::ptrdiff_t + +#include "task.h" // for task::suspend_point + +#if _WIN32 || _WIN64 +// declare prototypes of just the needed TLS functions, instead of +// including whole windows.h +#if __TBB_WIN8UI_SUPPORT +extern "C" __declspec(dllimport) unsigned long __stdcall FlsAlloc(void(__stdcall*)(void*)); +extern "C" __declspec(dllimport) int __stdcall FlsFree(unsigned long dwFlsIndex); +extern "C" __declspec(dllimport) int __stdcall FlsSetValue(unsigned long dwFlsIndex, void* lpFlsValue); +extern "C" __declspec(dllimport) void* __stdcall FlsGetValue(unsigned long dwFlsIndex); +#else +extern "C" __declspec(dllimport) unsigned long __stdcall TlsAlloc(void); +extern "C" __declspec(dllimport) int __stdcall TlsFree(unsigned long dwTlsIndex); +extern "C" __declspec(dllimport) int __stdcall TlsSetValue(unsigned long dwTlsIndex, void* lpTlsValue); +extern "C" __declspec(dllimport) void* __stdcall TlsGetValue(unsigned long dwTlsIndex); +#endif +#else +#include +#endif + +namespace tbb { +namespace detail { +namespace d1 { + +//! enum for selecting between single key and key-per-instance versions +enum ets_key_usage_type { + ets_key_per_instance + , ets_no_key +#if __TBB_RESUMABLE_TASKS + , ets_suspend_aware +#endif +}; + +// Forward declaration to use in internal classes +template +class enumerable_thread_specific; + +template +struct internal_ets_key_selector { + using key_type = std::thread::id; + static key_type current_key() { + return std::this_thread::get_id(); + } +}; + +// Intel Compiler on OSX cannot create atomics objects that instantiated from non-fundamental types +#if __INTEL_COMPILER && __APPLE__ +template<> +struct internal_ets_key_selector { + using key_type = std::size_t; + static key_type current_key() { + auto id = std::this_thread::get_id(); + return reinterpret_cast(id); + } +}; +#endif + +template +struct ets_key_selector : internal_ets_key_selector {}; + +#if __TBB_RESUMABLE_TASKS +template <> +struct ets_key_selector { + using key_type = suspend_point; + static key_type current_key() { + return r1::current_suspend_point(); + } +}; +#endif + +template +class ets_base : detail::no_copy { +protected: + using key_type = typename ets_key_selector::key_type; + +public: + struct slot; + struct array { + array* next; + std::size_t lg_size; + slot& at( std::size_t k ) { + return (reinterpret_cast(reinterpret_cast(this+1)))[k]; + } + std::size_t size() const { return std::size_t(1) << lg_size; } + std::size_t mask() const { return size() - 1; } + std::size_t start( std::size_t h ) const { + return h >> (8 * sizeof(std::size_t) - lg_size); + } + }; + struct slot { + std::atomic key; + void* ptr; + bool empty() const { return key.load(std::memory_order_relaxed) == key_type(); } + bool match( key_type k ) const { return key.load(std::memory_order_relaxed) == k; } + bool claim( key_type k ) { + // TODO: maybe claim ptr, because key_type is not guaranteed to fit into word size + key_type expected = key_type(); + return key.compare_exchange_strong(expected, k); + } + }; + +protected: + //! Root of linked list of arrays of decreasing size. + /** nullptr if and only if my_count==0. + Each array in the list is half the size of its predecessor. */ + std::atomic my_root; + std::atomic my_count; + + virtual void* create_local() = 0; + virtual void* create_array(std::size_t _size) = 0; // _size in bytes + virtual void free_array(void* ptr, std::size_t _size) = 0; // _size in bytes + + array* allocate( std::size_t lg_size ) { + std::size_t n = std::size_t(1) << lg_size; + array* a = static_cast(create_array(sizeof(array) + n * sizeof(slot))); + a->lg_size = lg_size; + std::memset( a + 1, 0, n * sizeof(slot) ); + return a; + } + void deallocate(array* a) { + std::size_t n = std::size_t(1) << (a->lg_size); + free_array( static_cast(a), std::size_t(sizeof(array) + n * sizeof(slot)) ); + } + + ets_base() : my_root{nullptr}, my_count{0} {} + virtual ~ets_base(); // g++ complains if this is not virtual + + void* table_lookup( bool& exists ); + void table_clear(); + // The following functions are not used in concurrent context, + // so we don't need synchronization and ITT annotations there. + template + void table_elementwise_copy( const ets_base& other, + void*(*add_element)(ets_base&, void*) ) { + __TBB_ASSERT(!my_root.load(std::memory_order_relaxed), nullptr); + __TBB_ASSERT(!my_count.load(std::memory_order_relaxed), nullptr); + if( !other.my_root.load(std::memory_order_relaxed) ) return; + array* root = allocate(other.my_root.load(std::memory_order_relaxed)->lg_size); + my_root.store(root, std::memory_order_relaxed); + root->next = nullptr; + my_count.store(other.my_count.load(std::memory_order_relaxed), std::memory_order_relaxed); + std::size_t mask = root->mask(); + for( array* r = other.my_root.load(std::memory_order_relaxed); r; r = r->next ) { + for( std::size_t i = 0; i < r->size(); ++i ) { + slot& s1 = r->at(i); + if( !s1.empty() ) { + for( std::size_t j = root->start(std::hash{}(s1.key.load(std::memory_order_relaxed))); ; j = (j+1)&mask ) { + slot& s2 = root->at(j); + if( s2.empty() ) { + s2.ptr = add_element(static_cast&>(*this), s1.ptr); + s2.key.store(s1.key.load(std::memory_order_relaxed), std::memory_order_relaxed); + break; + } + else if( s2.match(s1.key.load(std::memory_order_relaxed)) ) + break; + } + } + } + } + } + void table_swap( ets_base& other ) { + __TBB_ASSERT(this!=&other, "Don't swap an instance with itself"); + swap_atomics_relaxed(my_root, other.my_root); + swap_atomics_relaxed(my_count, other.my_count); + } +}; + +template +ets_base::~ets_base() { + __TBB_ASSERT(!my_root.load(std::memory_order_relaxed), nullptr); +} + +template +void ets_base::table_clear() { + while ( array* r = my_root.load(std::memory_order_relaxed) ) { + my_root.store(r->next, std::memory_order_relaxed); + deallocate(r); + } + my_count.store(0, std::memory_order_relaxed); +} + +template +void* ets_base::table_lookup( bool& exists ) { + const key_type k = ets_key_selector::current_key(); + + __TBB_ASSERT(k != key_type(), nullptr); + void* found; + std::size_t h = std::hash{}(k); + for( array* r = my_root.load(std::memory_order_acquire); r; r = r->next ) { + call_itt_notify(acquired,r); + std::size_t mask=r->mask(); + for(std::size_t i = r->start(h); ;i=(i+1)&mask) { + slot& s = r->at(i); + if( s.empty() ) break; + if( s.match(k) ) { + if( r == my_root.load(std::memory_order_acquire) ) { + // Success at top level + exists = true; + return s.ptr; + } else { + // Success at some other level. Need to insert at top level. + exists = true; + found = s.ptr; + goto insert; + } + } + } + } + // Key does not yet exist. The density of slots in the table does not exceed 0.5, + // for if this will occur a new table is allocated with double the current table + // size, which is swapped in as the new root table. So an empty slot is guaranteed. + exists = false; + found = create_local(); + { + std::size_t c = ++my_count; + array* r = my_root.load(std::memory_order_acquire); + call_itt_notify(acquired,r); + if( !r || c > r->size()/2 ) { + std::size_t s = r ? r->lg_size : 2; + while( c > std::size_t(1)<<(s-1) ) ++s; + array* a = allocate(s); + for(;;) { + a->next = r; + call_itt_notify(releasing,a); + array* new_r = r; + if( my_root.compare_exchange_strong(new_r, a) ) break; + call_itt_notify(acquired, new_r); + __TBB_ASSERT(new_r != nullptr, nullptr); + if( new_r->lg_size >= s ) { + // Another thread inserted an equal or bigger array, so our array is superfluous. + deallocate(a); + break; + } + r = new_r; + } + } + } + insert: + // Whether a slot has been found in an older table, or if it has been inserted at this level, + // it has already been accounted for in the total. Guaranteed to be room for it, and it is + // not present, so search for empty slot and use it. + array* ir = my_root.load(std::memory_order_acquire); + call_itt_notify(acquired, ir); + std::size_t mask = ir->mask(); + for(std::size_t i = ir->start(h);; i = (i+1)&mask) { + slot& s = ir->at(i); + if( s.empty() ) { + if( s.claim(k) ) { + s.ptr = found; + return found; + } + } + } +} + +//! Specialization that exploits native TLS +template <> +class ets_base: public ets_base { + using super = ets_base; +#if _WIN32||_WIN64 +#if __TBB_WIN8UI_SUPPORT + using tls_key_t = unsigned long; + void create_key() { my_key = FlsAlloc(nullptr); } + void destroy_key() { FlsFree(my_key); } + void set_tls(void * value) { FlsSetValue(my_key, value); } + void* get_tls() { return (void *)FlsGetValue(my_key); } +#else + using tls_key_t = unsigned long; + void create_key() { my_key = TlsAlloc(); } + void destroy_key() { TlsFree(my_key); } + void set_tls(void * value) { TlsSetValue(my_key, value); } + void* get_tls() { return (void *)TlsGetValue(my_key); } +#endif +#else + using tls_key_t = pthread_key_t; + void create_key() { pthread_key_create(&my_key, nullptr); } + void destroy_key() { pthread_key_delete(my_key); } + void set_tls( void * value ) const { pthread_setspecific(my_key, value); } + void* get_tls() const { return pthread_getspecific(my_key); } +#endif + tls_key_t my_key; + virtual void* create_local() override = 0; + virtual void* create_array(std::size_t _size) override = 0; // _size in bytes + virtual void free_array(void* ptr, std::size_t _size) override = 0; // size in bytes +protected: + ets_base() {create_key();} + ~ets_base() {destroy_key();} + void* table_lookup( bool& exists ) { + void* found = get_tls(); + if( found ) { + exists=true; + } else { + found = super::table_lookup(exists); + set_tls(found); + } + return found; + } + void table_clear() { + destroy_key(); + create_key(); + super::table_clear(); + } + void table_swap( ets_base& other ) { + using std::swap; + __TBB_ASSERT(this!=&other, "Don't swap an instance with itself"); + swap(my_key, other.my_key); + super::table_swap(other); + } +}; + +//! Random access iterator for traversing the thread local copies. +template< typename Container, typename Value > +class enumerable_thread_specific_iterator +{ + //! current position in the concurrent_vector + + Container *my_container; + typename Container::size_type my_index; + mutable Value *my_value; + + template + friend bool operator==( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ); + + template + friend bool operator<( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ); + + template + friend std::ptrdiff_t operator-( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ); + + template + friend class enumerable_thread_specific_iterator; + +public: + //! STL support + using difference_type = std::ptrdiff_t; + using value_type = Value; + using pointer = Value*; + using reference = Value&; + using iterator_category = std::random_access_iterator_tag; + + enumerable_thread_specific_iterator( const Container &container, typename Container::size_type index ) : + my_container(&const_cast(container)), my_index(index), my_value(nullptr) {} + + //! Default constructor + enumerable_thread_specific_iterator() : my_container(nullptr), my_index(0), my_value(nullptr) {} + + template + enumerable_thread_specific_iterator( const enumerable_thread_specific_iterator& other ) : + my_container( other.my_container ), my_index( other.my_index), my_value( const_cast(other.my_value) ) {} + + enumerable_thread_specific_iterator operator+( std::ptrdiff_t offset ) const { + return enumerable_thread_specific_iterator(*my_container, my_index + offset); + } + + friend enumerable_thread_specific_iterator operator+( std::ptrdiff_t offset, enumerable_thread_specific_iterator v ) { + return enumerable_thread_specific_iterator(*v.my_container, v.my_index + offset); + } + + enumerable_thread_specific_iterator &operator+=( std::ptrdiff_t offset ) { + my_index += offset; + my_value = nullptr; + return *this; + } + + enumerable_thread_specific_iterator operator-( std::ptrdiff_t offset ) const { + return enumerable_thread_specific_iterator( *my_container, my_index-offset ); + } + + enumerable_thread_specific_iterator &operator-=( std::ptrdiff_t offset ) { + my_index -= offset; + my_value = nullptr; + return *this; + } + + Value& operator*() const { + Value* value = my_value; + if( !value ) { + value = my_value = (*my_container)[my_index].value(); + } + __TBB_ASSERT( value==(*my_container)[my_index].value(), "corrupt cache" ); + return *value; + } + + Value& operator[]( std::ptrdiff_t k ) const { + return *(*my_container)[my_index + k].value(); + } + + Value* operator->() const {return &operator*();} + + enumerable_thread_specific_iterator& operator++() { + ++my_index; + my_value = nullptr; + return *this; + } + + enumerable_thread_specific_iterator& operator--() { + --my_index; + my_value = nullptr; + return *this; + } + + //! Post increment + enumerable_thread_specific_iterator operator++(int) { + enumerable_thread_specific_iterator result = *this; + ++my_index; + my_value = nullptr; + return result; + } + + //! Post decrement + enumerable_thread_specific_iterator operator--(int) { + enumerable_thread_specific_iterator result = *this; + --my_index; + my_value = nullptr; + return result; + } +}; + +template +bool operator==( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ) { + return i.my_index == j.my_index && i.my_container == j.my_container; +} + +template +bool operator!=( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ) { + return !(i==j); +} + +template +bool operator<( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ) { + return i.my_index +bool operator>( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ) { + return j +bool operator>=( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ) { + return !(i +bool operator<=( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ) { + return !(j +std::ptrdiff_t operator-( const enumerable_thread_specific_iterator& i, + const enumerable_thread_specific_iterator& j ) { + return i.my_index-j.my_index; +} + +template +class segmented_iterator +{ + template + friend bool operator==(const segmented_iterator& i, const segmented_iterator& j); + + template + friend bool operator!=(const segmented_iterator& i, const segmented_iterator& j); + + template + friend class segmented_iterator; + +public: + segmented_iterator() {my_segcont = nullptr;} + + segmented_iterator( const SegmentedContainer& _segmented_container ) : + my_segcont(const_cast(&_segmented_container)), + outer_iter(my_segcont->end()) { } + + ~segmented_iterator() {} + + using InnerContainer = typename SegmentedContainer::value_type; + using inner_iterator = typename InnerContainer::iterator; + using outer_iterator = typename SegmentedContainer::iterator; + + // STL support + // TODO: inherit all types from segmented container? + using difference_type = std::ptrdiff_t; + using value_type = Value; + using size_type = typename SegmentedContainer::size_type; + using pointer = Value*; + using reference = Value&; + using iterator_category = std::input_iterator_tag; + + // Copy Constructor + template + segmented_iterator(const segmented_iterator& other) : + my_segcont(other.my_segcont), + outer_iter(other.outer_iter), + // can we assign a default-constructed iterator to inner if we're at the end? + inner_iter(other.inner_iter) + {} + + // assignment + template + segmented_iterator& operator=( const segmented_iterator& other) { + my_segcont = other.my_segcont; + outer_iter = other.outer_iter; + if(outer_iter != my_segcont->end()) inner_iter = other.inner_iter; + return *this; + } + + // allow assignment of outer iterator to segmented iterator. Once it is + // assigned, move forward until a non-empty inner container is found or + // the end of the outer container is reached. + segmented_iterator& operator=(const outer_iterator& new_outer_iter) { + __TBB_ASSERT(my_segcont != nullptr, nullptr); + // check that this iterator points to something inside the segmented container + for(outer_iter = new_outer_iter ;outer_iter!=my_segcont->end(); ++outer_iter) { + if( !outer_iter->empty() ) { + inner_iter = outer_iter->begin(); + break; + } + } + return *this; + } + + // pre-increment + segmented_iterator& operator++() { + advance_me(); + return *this; + } + + // post-increment + segmented_iterator operator++(int) { + segmented_iterator tmp = *this; + operator++(); + return tmp; + } + + bool operator==(const outer_iterator& other_outer) const { + __TBB_ASSERT(my_segcont != nullptr, nullptr); + return (outer_iter == other_outer && + (outer_iter == my_segcont->end() || inner_iter == outer_iter->begin())); + } + + bool operator!=(const outer_iterator& other_outer) const { + return !operator==(other_outer); + + } + + // (i)* RHS + reference operator*() const { + __TBB_ASSERT(my_segcont != nullptr, nullptr); + __TBB_ASSERT(outer_iter != my_segcont->end(), "Dereferencing a pointer at end of container"); + __TBB_ASSERT(inner_iter != outer_iter->end(), nullptr); // should never happen + return *inner_iter; + } + + // i-> + pointer operator->() const { return &operator*();} + +private: + SegmentedContainer* my_segcont; + outer_iterator outer_iter; + inner_iterator inner_iter; + + void advance_me() { + __TBB_ASSERT(my_segcont != nullptr, nullptr); + __TBB_ASSERT(outer_iter != my_segcont->end(), nullptr); // not true if there are no inner containers + __TBB_ASSERT(inner_iter != outer_iter->end(), nullptr); // not true if the inner containers are all empty. + ++inner_iter; + while(inner_iter == outer_iter->end() && ++outer_iter != my_segcont->end()) { + inner_iter = outer_iter->begin(); + } + } +}; // segmented_iterator + +template +bool operator==( const segmented_iterator& i, + const segmented_iterator& j ) { + if(i.my_segcont != j.my_segcont) return false; + if(i.my_segcont == nullptr) return true; + if(i.outer_iter != j.outer_iter) return false; + if(i.outer_iter == i.my_segcont->end()) return true; + return i.inner_iter == j.inner_iter; +} + +// != +template +bool operator!=( const segmented_iterator& i, + const segmented_iterator& j ) { + return !(i==j); +} + +template +struct construct_by_default: no_assign { + void construct(void*where) {new(where) T();} // C++ note: the () in T() ensure zero initialization. + construct_by_default( int ) {} +}; + +template +struct construct_by_exemplar: no_assign { + const T exemplar; + void construct(void*where) {new(where) T(exemplar);} + construct_by_exemplar( const T& t ) : exemplar(t) {} + construct_by_exemplar( T&& t ) : exemplar(std::move(t)) {} +}; + +template +struct construct_by_finit: no_assign { + Finit f; + void construct(void* where) {new(where) T(f());} + construct_by_finit( Finit&& f_ ) : f(std::move(f_)) {} +}; + +template +struct construct_by_args: no_assign { + stored_pack pack; + void construct(void* where) { + call( [where](const typename std::decay

::type&... args ){ + new(where) T(args...); + }, pack ); + } + construct_by_args( P&& ... args ) : pack(std::forward

(args)...) {} +}; + +// storage for initialization function pointer +// TODO: consider removing the template parameter T here and in callback_leaf +class callback_base { +public: + // Clone *this + virtual callback_base* clone() const = 0; + // Destruct and free *this + virtual void destroy() = 0; + // Need virtual destructor to satisfy GCC compiler warning + virtual ~callback_base() { } + // Construct T at where + virtual void construct(void* where) = 0; +}; + +template +class callback_leaf: public callback_base, Constructor { + template callback_leaf( P&& ... params ) : Constructor(std::forward

(params)...) {} + // TODO: make the construction/destruction consistent (use allocator.construct/destroy) + using my_allocator_type = typename tbb::tbb_allocator; + + callback_base* clone() const override { + return make(*this); + } + + void destroy() override { + my_allocator_type alloc; + tbb::detail::allocator_traits::destroy(alloc, this); + tbb::detail::allocator_traits::deallocate(alloc, this, 1); + } + + void construct(void* where) override { + Constructor::construct(where); + } + +public: + template + static callback_base* make( P&& ... params ) { + void* where = my_allocator_type().allocate(1); + return new(where) callback_leaf( std::forward

(params)... ); + } +}; + +//! Template for recording construction of objects in table +/** All maintenance of the space will be done explicitly on push_back, + and all thread local copies must be destroyed before the concurrent + vector is deleted. + + The flag is_built is initialized to false. When the local is + successfully-constructed, set the flag to true or call value_committed(). + If the constructor throws, the flag will be false. +*/ +template +struct ets_element { + detail::aligned_space my_space; + bool is_built; + ets_element() { is_built = false; } // not currently-built + U* value() { return my_space.begin(); } + U* value_committed() { is_built = true; return my_space.begin(); } + ~ets_element() { + if(is_built) { + my_space.begin()->~U(); + is_built = false; + } + } +}; + +// A predicate that can be used for a compile-time compatibility check of ETS instances +// Ideally, it should have been declared inside the ETS class, but unfortunately +// in that case VS2013 does not enable the variadic constructor. +template struct is_compatible_ets : std::false_type {}; +template +struct is_compatible_ets< T, enumerable_thread_specific > : std::is_same {}; + +// A predicate that checks whether, for a variable 'foo' of type T, foo() is a valid expression +template using has_empty_braces_operator = decltype(std::declval()()); +template using is_callable_no_args = supports; + +//! The enumerable_thread_specific container +/** enumerable_thread_specific has the following properties: + - thread-local copies are lazily created, with default, exemplar or function initialization. + - thread-local copies do not move (during lifetime, and excepting clear()) so the address of a copy is invariant. + - the contained objects need not have operator=() defined if combine is not used. + - enumerable_thread_specific containers may be copy-constructed or assigned. + - thread-local copies can be managed by hash-table, or can be accessed via TLS storage for speed. + - outside of parallel contexts, the contents of all thread-local copies are accessible by iterator or using combine or combine_each methods + +@par Segmented iterator + When the thread-local objects are containers with input_iterators defined, a segmented iterator may + be used to iterate over all the elements of all thread-local copies. + +@par combine and combine_each + - Both methods are defined for enumerable_thread_specific. + - combine() requires the type T have operator=() defined. + - neither method modifies the contents of the object (though there is no guarantee that the applied methods do not modify the object.) + - Both are evaluated in serial context (the methods are assumed to be non-benign.) + +@ingroup containers */ +template , + ets_key_usage_type ETS_key_type=ets_no_key > +class enumerable_thread_specific: ets_base { + + template friend class enumerable_thread_specific; + + using padded_element = padded>; + + //! A generic range, used to create range objects from the iterators + template + class generic_range_type: public blocked_range { + public: + using value_type = T; + using reference = T&; + using const_reference = const T&; + using iterator = I; + using difference_type = std::ptrdiff_t; + + generic_range_type( I begin_, I end_, std::size_t grainsize_ = 1) : blocked_range(begin_,end_,grainsize_) {} + template + generic_range_type( const generic_range_type& r) : blocked_range(r.begin(),r.end(),r.grainsize()) {} + generic_range_type( generic_range_type& r, split ) : blocked_range(r,split()) {} + }; + + using allocator_traits_type = tbb::detail::allocator_traits; + + using padded_allocator_type = typename allocator_traits_type::template rebind_alloc; + using internal_collection_type = tbb::concurrent_vector< padded_element, padded_allocator_type >; + + callback_base *my_construct_callback; + + internal_collection_type my_locals; + + // TODO: consider unifying the callback mechanism for all create_local* methods below + // (likely non-compatible and requires interface version increase) + void* create_local() override { + padded_element& lref = *my_locals.grow_by(1); + my_construct_callback->construct(lref.value()); + return lref.value_committed(); + } + + static void* create_local_by_copy( ets_base& base, void* p ) { + enumerable_thread_specific& ets = static_cast(base); + padded_element& lref = *ets.my_locals.grow_by(1); + new(lref.value()) T(*static_cast(p)); + return lref.value_committed(); + } + + static void* create_local_by_move( ets_base& base, void* p ) { + enumerable_thread_specific& ets = static_cast(base); + padded_element& lref = *ets.my_locals.grow_by(1); + new(lref.value()) T(std::move(*static_cast(p))); + return lref.value_committed(); + } + + using array_allocator_type = typename allocator_traits_type::template rebind_alloc; + + // _size is in bytes + void* create_array(std::size_t _size) override { + std::size_t nelements = (_size + sizeof(uintptr_t) -1) / sizeof(uintptr_t); + return array_allocator_type().allocate(nelements); + } + + void free_array( void* _ptr, std::size_t _size) override { + std::size_t nelements = (_size + sizeof(uintptr_t) -1) / sizeof(uintptr_t); + array_allocator_type().deallocate( reinterpret_cast(_ptr),nelements); + } + +public: + + //! Basic types + using value_type = T; + using allocator_type = Allocator; + using size_type = typename internal_collection_type::size_type; + using difference_type = typename internal_collection_type::difference_type; + using reference = value_type&; + using const_reference = const value_type&; + + using pointer = typename allocator_traits_type::pointer; + using const_pointer = typename allocator_traits_type::const_pointer; + + // Iterator types + using iterator = enumerable_thread_specific_iterator; + using const_iterator = enumerable_thread_specific_iterator; + + // Parallel range types + using range_type = generic_range_type; + using const_range_type = generic_range_type; + + //! Default constructor. Each local instance of T is default constructed. + enumerable_thread_specific() : my_construct_callback( + callback_leaf >::make(/*dummy argument*/0) + ){} + + //! Constructor with initializer functor. Each local instance of T is constructed by T(finit()). + template ::type>::value>::type> + explicit enumerable_thread_specific( Finit finit ) : my_construct_callback( + callback_leaf >::make( std::move(finit) ) + ){} + + //! Constructor with exemplar. Each local instance of T is copy-constructed from the exemplar. + explicit enumerable_thread_specific( const T& exemplar ) : my_construct_callback( + callback_leaf >::make( exemplar ) + ){} + + explicit enumerable_thread_specific( T&& exemplar ) : my_construct_callback( + callback_leaf >::make( std::move(exemplar) ) + ){} + + //! Variadic constructor with initializer arguments. Each local instance of T is constructed by T(args...) + template ::type>::value + && !is_compatible_ets::type>::value + && !std::is_same::type>::value + >::type> + enumerable_thread_specific( P1&& arg1, P&& ... args ) : my_construct_callback( + callback_leaf >::make( std::forward(arg1), std::forward

(args)... ) + ){} + + //! Destructor + ~enumerable_thread_specific() { + if(my_construct_callback) my_construct_callback->destroy(); + // Deallocate the hash table before overridden free_array() becomes inaccessible + this->ets_base::table_clear(); + } + + //! returns reference to local, discarding exists + reference local() { + bool exists; + return local(exists); + } + + //! Returns reference to calling thread's local copy, creating one if necessary + reference local(bool& exists) { + void* ptr = this->table_lookup(exists); + return *(T*)ptr; + } + + //! Get the number of local copies + size_type size() const { return my_locals.size(); } + + //! true if there have been no local copies created + bool empty() const { return my_locals.empty(); } + + //! begin iterator + iterator begin() { return iterator( my_locals, 0 ); } + //! end iterator + iterator end() { return iterator(my_locals, my_locals.size() ); } + + //! begin const iterator + const_iterator begin() const { return const_iterator(my_locals, 0); } + + //! end const iterator + const_iterator end() const { return const_iterator(my_locals, my_locals.size()); } + + //! Get range for parallel algorithms + range_type range( std::size_t grainsize=1 ) { return range_type( begin(), end(), grainsize ); } + + //! Get const range for parallel algorithms + const_range_type range( std::size_t grainsize=1 ) const { return const_range_type( begin(), end(), grainsize ); } + + //! Destroys local copies + void clear() { + my_locals.clear(); + this->table_clear(); + // callback is not destroyed + } + +private: + template + void internal_copy(const enumerable_thread_specific& other) { + // this tests is_compatible_ets + static_assert( (is_compatible_ets::type>::value), "is_compatible_ets fails" ); + // Initialize my_construct_callback first, so that it is valid even if rest of this routine throws an exception. + my_construct_callback = other.my_construct_callback->clone(); + __TBB_ASSERT(my_locals.size()==0, nullptr); + my_locals.reserve(other.size()); + this->table_elementwise_copy( other, create_local_by_copy ); + } + + void internal_swap(enumerable_thread_specific& other) { + using std::swap; + __TBB_ASSERT( this!=&other, nullptr); + swap(my_construct_callback, other.my_construct_callback); + // concurrent_vector::swap() preserves storage space, + // so addresses to the vector kept in ETS hash table remain valid. + swap(my_locals, other.my_locals); + this->ets_base::table_swap(other); + } + + template + void internal_move(enumerable_thread_specific&& other) { + static_assert( (is_compatible_ets::type>::value), "is_compatible_ets fails" ); + my_construct_callback = other.my_construct_callback; + other.my_construct_callback = nullptr; + __TBB_ASSERT(my_locals.size()==0, nullptr); + my_locals.reserve(other.size()); + this->table_elementwise_copy( other, create_local_by_move ); + } + +public: + enumerable_thread_specific( const enumerable_thread_specific& other ) + : ets_base() /* prevents GCC warnings with -Wextra */ + { + internal_copy(other); + } + + template + enumerable_thread_specific( const enumerable_thread_specific& other ) + { + internal_copy(other); + } + + enumerable_thread_specific( enumerable_thread_specific&& other ) : my_construct_callback() + { + // TODO: use internal_move correctly here + internal_swap(other); + } + + template + enumerable_thread_specific( enumerable_thread_specific&& other ) : my_construct_callback() + { + internal_move(std::move(other)); + } + + enumerable_thread_specific& operator=( const enumerable_thread_specific& other ) + { + if( this != &other ) { + this->clear(); + my_construct_callback->destroy(); + internal_copy( other ); + } + return *this; + } + + template + enumerable_thread_specific& operator=( const enumerable_thread_specific& other ) + { + __TBB_ASSERT( static_cast(this)!=static_cast(&other), nullptr); // Objects of different types + this->clear(); + my_construct_callback->destroy(); + internal_copy(other); + return *this; + } + + enumerable_thread_specific& operator=( enumerable_thread_specific&& other ) + { + if( this != &other ) { + // TODO: use internal_move correctly here + internal_swap(other); + } + return *this; + } + + template + enumerable_thread_specific& operator=( enumerable_thread_specific&& other ) + { + __TBB_ASSERT( static_cast(this)!=static_cast(&other), nullptr); // Objects of different types + this->clear(); + my_construct_callback->destroy(); + internal_move(std::move(other)); + return *this; + } + + // CombineFunc has signature T(T,T) or T(const T&, const T&) + template + T combine(CombineFunc f_combine) { + if(begin() == end()) { + ets_element location; + my_construct_callback->construct(location.value()); + return *location.value_committed(); + } + const_iterator ci = begin(); + T my_result = *ci; + while(++ci != end()) + my_result = f_combine( my_result, *ci ); + return my_result; + } + + // combine_func_t takes T by value or by [const] reference, and returns nothing + template + void combine_each(CombineFunc f_combine) { + for(iterator ci = begin(); ci != end(); ++ci) { + f_combine( *ci ); + } + } + +}; // enumerable_thread_specific + +template< typename Container > +class flattened2d { + // This intermediate typedef is to address issues with VC7.1 compilers + using conval_type = typename Container::value_type; + +public: + //! Basic types + using size_type = typename conval_type::size_type; + using difference_type = typename conval_type::difference_type; + using allocator_type = typename conval_type::allocator_type; + using value_type = typename conval_type::value_type; + using reference = typename conval_type::reference; + using const_reference = typename conval_type::const_reference; + using pointer = typename conval_type::pointer; + using const_pointer = typename conval_type::const_pointer; + + using iterator = segmented_iterator; + using const_iterator = segmented_iterator; + + flattened2d( const Container &c, typename Container::const_iterator b, typename Container::const_iterator e ) : + my_container(const_cast(&c)), my_begin(b), my_end(e) { } + + explicit flattened2d( const Container &c ) : + my_container(const_cast(&c)), my_begin(c.begin()), my_end(c.end()) { } + + iterator begin() { return iterator(*my_container) = my_begin; } + iterator end() { return iterator(*my_container) = my_end; } + const_iterator begin() const { return const_iterator(*my_container) = my_begin; } + const_iterator end() const { return const_iterator(*my_container) = my_end; } + + size_type size() const { + size_type tot_size = 0; + for(typename Container::const_iterator i = my_begin; i != my_end; ++i) { + tot_size += i->size(); + } + return tot_size; + } + +private: + Container *my_container; + typename Container::const_iterator my_begin; + typename Container::const_iterator my_end; +}; + +template +flattened2d flatten2d(const Container &c, const typename Container::const_iterator b, const typename Container::const_iterator e) { + return flattened2d(c, b, e); +} + +template +flattened2d flatten2d(const Container &c) { + return flattened2d(c); +} + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::enumerable_thread_specific; +using detail::d1::flattened2d; +using detail::d1::flatten2d; +// ets enum keys +using detail::d1::ets_key_usage_type; +using detail::d1::ets_key_per_instance; +using detail::d1::ets_no_key; +#if __TBB_RESUMABLE_TASKS +using detail::d1::ets_suspend_aware; +#endif +} // inline namespace v1 + +} // namespace tbb + +#endif // __TBB_enumerable_thread_specific_H + diff --git a/src/3rd party/oneapi/tbb/flow_graph.h b/src/3rd party/oneapi/tbb/flow_graph.h new file mode 100644 index 0000000000..46fce47a22 --- /dev/null +++ b/src/3rd party/oneapi/tbb/flow_graph.h @@ -0,0 +1,3393 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + Copyright (c) 2025 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_flow_graph_H +#define __TBB_flow_graph_H + +#include +#include +#include + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "spin_mutex.h" +#include "null_mutex.h" +#include "spin_rw_mutex.h" +#include "null_rw_mutex.h" +#include "detail/_pipeline_filters.h" +#include "detail/_task.h" +#include "detail/_small_object_pool.h" +#include "cache_aligned_allocator.h" +#include "detail/_exception.h" +#include "detail/_template_helpers.h" +#include "detail/_aggregator.h" +#include "detail/_allocator_traits.h" +#include "detail/_utils.h" +#include "profiling.h" +#include "task_arena.h" + +#if TBB_USE_PROFILING_TOOLS && ( __unix__ || __APPLE__ ) + #if __INTEL_COMPILER + // Disabled warning "routine is both inline and noinline" + #pragma warning (push) + #pragma warning( disable: 2196 ) + #endif + #define __TBB_NOINLINE_SYM __attribute__((noinline)) +#else + #define __TBB_NOINLINE_SYM +#endif + +#include +#include +#include +#include +#if __TBB_CPP20_CONCEPTS_PRESENT +#include +#endif + +/** @file + \brief The graph related classes and functions + + There are some applications that best express dependencies as messages + passed between nodes in a graph. These messages may contain data or + simply act as signals that a predecessors has completed. The graph + class and its associated node classes can be used to express such + applications. +*/ + +namespace tbb { +namespace detail { + +namespace d2 { + +//! An enumeration the provides the two most common concurrency levels: unlimited and serial +enum concurrency { unlimited = 0, serial = 1 }; + +//! A generic null type +struct null_type {}; + +//! An empty class used for messages that mean "I'm done" +class continue_msg {}; + +} // namespace d2 + +#if __TBB_CPP20_CONCEPTS_PRESENT +inline namespace d0 { + +template +concept node_body_return_type = std::same_as || + std::convertible_to; + +// TODO: consider using std::invocable here +template +concept continue_node_body = std::copy_constructible && + requires( Body& body, const tbb::detail::d2::continue_msg& v ) { + { body(v) } -> node_body_return_type; + }; + +template +concept function_node_body = std::copy_constructible && + std::invocable && + node_body_return_type, Output>; + +template +concept join_node_function_object = std::copy_constructible && + std::invocable && + std::convertible_to, Key>; + +template +concept input_node_body = std::copy_constructible && + requires( Body& body, tbb::detail::d1::flow_control& fc ) { + { body(fc) } -> adaptive_same_as; + }; + +template +concept multifunction_node_body = std::copy_constructible && + std::invocable; + +template +concept sequencer = std::copy_constructible && + std::invocable && + std::convertible_to, std::size_t>; + +template +concept async_node_body = std::copy_constructible && + std::invocable; + +} // inline namespace d0 +#endif // __TBB_CPP20_CONCEPTS_PRESENT + +namespace d2 { + +//! Forward declaration section +template< typename T > class sender; +template< typename T > class receiver; +class continue_receiver; + +template< typename T, typename U > class limiter_node; // needed for resetting decrementer +template< typename T > class overwrite_node; // needed for the forward friend declaration to work in GCC < 8 + +template class successor_cache; +template class broadcast_cache; +template class round_robin_cache; +template class predecessor_cache; +template class reservable_predecessor_cache; + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET +namespace order { +struct following; +struct preceding; +} +template struct node_set; +#endif + + +} // namespace d2 +} // namespace detail +} // namespace tbb + +//! The graph class +#include "detail/_flow_graph_impl.h" + +namespace tbb { +namespace detail { +namespace d2 { + +inline std::pair order_tasks(graph_task* first, graph_task* second) { + if (second->priority > first->priority) + return std::make_pair(second, first); + return std::make_pair(first, second); +} + +// submit task if necessary. Returns the non-enqueued task if there is one. +inline graph_task* combine_tasks(graph& g, graph_task* left, graph_task* right) { + // if no RHS task, don't change left. + if (right == nullptr) return left; + // right != nullptr + if (left == nullptr) return right; + if (left == SUCCESSFULLY_ENQUEUED) return right; + // left contains a task + if (right != SUCCESSFULLY_ENQUEUED) { + // both are valid tasks + auto tasks_pair = order_tasks(left, right); + spawn_in_graph_arena(g, *tasks_pair.first); + return tasks_pair.second; + } + return left; +} + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +class message_metainfo { +public: + using waiters_type = std::forward_list; + + message_metainfo() = default; + + message_metainfo(const waiters_type& waiters) : my_waiters(waiters) {} + message_metainfo(waiters_type&& waiters) : my_waiters(std::move(waiters)) {} + + const waiters_type& waiters() const & { return my_waiters; } + waiters_type&& waiters() && { return std::move(my_waiters); } + + bool empty() const { return my_waiters.empty(); } + + void merge(const message_metainfo& other) { + // TODO: should we avoid duplications on merging + my_waiters.insert_after(my_waiters.before_begin(), + other.waiters().begin(), + other.waiters().end()); + } +private: + waiters_type my_waiters; +}; // class message_metainfo + +#define __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo) , metainfo + +#else +#define __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo) +#endif // __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + +//! Pure virtual template class that defines a sender of messages of type T +template< typename T > +class sender { +public: + virtual ~sender() {} + + //! Request an item from the sender + virtual bool try_get( T & ) { return false; } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + virtual bool try_get( T &, message_metainfo& ) { return false; } +#endif + + //! Reserves an item in the sender + virtual bool try_reserve( T & ) { return false; } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + virtual bool try_reserve( T &, message_metainfo& ) { return false; } +#endif + + //! Releases the reserved item + virtual bool try_release( ) { return false; } + + //! Consumes the reserved item + virtual bool try_consume( ) { return false; } + +protected: + //! The output type of this sender + typedef T output_type; + + //! The successor type for this node + typedef receiver successor_type; + + //! Add a new successor to this node + virtual bool register_successor( successor_type &r ) = 0; + + //! Removes a successor from this node + virtual bool remove_successor( successor_type &r ) = 0; + + template + friend bool register_successor(sender& s, receiver& r); + + template + friend bool remove_successor (sender& s, receiver& r); +}; // class sender + +template +bool register_successor(sender& s, receiver& r) { + return s.register_successor(r); +} + +template +bool remove_successor(sender& s, receiver& r) { + return s.remove_successor(r); +} + +//! Pure virtual template class that defines a receiver of messages of type T +template< typename T > +class receiver { +private: + template + bool internal_try_put(const T& t, TryPutTaskArgs&&... args) { + graph_task* res = try_put_task(t, std::forward(args)...); + if (!res) return false; + if (res != SUCCESSFULLY_ENQUEUED) spawn_in_graph_arena(graph_reference(), *res); + return true; + } + +public: + //! Destructor + virtual ~receiver() {} + + //! Put an item to the receiver + bool try_put( const T& t ) { + return internal_try_put(t); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + //! Put an item to the receiver and wait for completion + bool try_put_and_wait( const T& t ) { + // Since try_put_and_wait is a blocking call, it is safe to create wait_context on stack + d1::wait_context_vertex msg_wait_vertex{}; + + bool res = internal_try_put(t, message_metainfo{message_metainfo::waiters_type{&msg_wait_vertex}}); + if (res) { + __TBB_ASSERT(graph_reference().my_context != nullptr, "No wait_context associated with the Flow Graph"); + d1::wait(msg_wait_vertex.get_context(), *graph_reference().my_context); + } + return res; + } +#endif + + //! put item to successor; return task to run the successor if possible. +protected: + //! The input type of this receiver + typedef T input_type; + + //! The predecessor type for this node + typedef sender predecessor_type; + + template< typename R, typename B > friend class run_and_put_task; + template< typename X, typename Y > friend class broadcast_cache; + template< typename X, typename Y > friend class round_robin_cache; + virtual graph_task *try_put_task(const T& t) = 0; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + virtual graph_task *try_put_task(const T& t, const message_metainfo&) = 0; +#endif + virtual graph& graph_reference() const = 0; + + template friend class successor_cache; + template< typename TTT > friend class overwrite_node; + virtual bool is_continue_receiver() { return false; } + + // TODO revamp: reconsider the inheritance and move node priority out of receiver + virtual node_priority_t priority() const { return no_priority; } + + //! Add a predecessor to the node + virtual bool register_predecessor( predecessor_type & ) { return false; } + + //! Remove a predecessor from the node + virtual bool remove_predecessor( predecessor_type & ) { return false; } + + template + friend bool register_predecessor(receiver& r, sender& s); + template + friend bool remove_predecessor (receiver& r, sender& s); +}; // class receiver + +template +bool register_predecessor(receiver& r, sender& s) { + return r.register_predecessor(s); +} + +template +bool remove_predecessor(receiver& r, sender& s) { + return r.remove_predecessor(s); +} + +//! Base class for receivers of completion messages +/** These receivers automatically reset, but cannot be explicitly waited on */ +class continue_receiver : public receiver< continue_msg > { +protected: + + //! Constructor + explicit continue_receiver( int number_of_predecessors, node_priority_t a_priority ) { + my_predecessor_count = my_initial_predecessor_count = number_of_predecessors; + my_current_count = 0; + my_priority = a_priority; + } + + //! Copy constructor + continue_receiver( const continue_receiver& src ) : receiver() { + my_predecessor_count = my_initial_predecessor_count = src.my_initial_predecessor_count; + my_current_count = 0; + my_priority = src.my_priority; + } + + //! Increments the trigger threshold + bool register_predecessor( predecessor_type & ) override { + spin_mutex::scoped_lock l(my_mutex); + ++my_predecessor_count; + return true; + } + + //! Decrements the trigger threshold + /** Does not check to see if the removal of the predecessor now makes the current count + exceed the new threshold. So removing a predecessor while the graph is active can cause + unexpected results. */ + bool remove_predecessor( predecessor_type & ) override { + spin_mutex::scoped_lock l(my_mutex); + --my_predecessor_count; + return true; + } + + //! The input type + typedef continue_msg input_type; + + //! The predecessor type for this node + typedef receiver::predecessor_type predecessor_type; + + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + +private: + // execute body is supposed to be too small to create a task for. + graph_task* try_put_task_impl( const input_type& __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo) ) { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo predecessor_metainfo; +#endif + { + spin_mutex::scoped_lock l(my_mutex); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + // Prolong the wait and store the metainfo until receiving signals from all the predecessors + for (auto waiter : metainfo.waiters()) { + waiter->reserve(1); + } + my_current_metainfo.merge(metainfo); +#endif + if ( ++my_current_count < my_predecessor_count ) + return SUCCESSFULLY_ENQUEUED; + else { + my_current_count = 0; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + predecessor_metainfo = my_current_metainfo; + my_current_metainfo = message_metainfo{}; +#endif + } + } +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* res = execute(predecessor_metainfo); + for (auto waiter : predecessor_metainfo.waiters()) { + waiter->release(1); + } +#else + graph_task* res = execute(); +#endif + return res? res : SUCCESSFULLY_ENQUEUED; + } + +protected: + graph_task* try_put_task( const input_type& input ) override { + return try_put_task_impl(input __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task( const input_type& input, const message_metainfo& metainfo ) override { + return try_put_task_impl(input, metainfo); + } +#endif + + spin_mutex my_mutex; + int my_predecessor_count; + int my_current_count; + int my_initial_predecessor_count; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo my_current_metainfo; +#endif + node_priority_t my_priority; + // the friend declaration in the base class did not eliminate the "protected class" + // error in gcc 4.1.2 + template friend class limiter_node; + + virtual void reset_receiver( reset_flags f ) { + my_current_count = 0; + if (f & rf_clear_edges) { + my_predecessor_count = my_initial_predecessor_count; + } + } + + //! Does whatever should happen when the threshold is reached + /** This should be very fast or else spawn a task. This is + called while the sender is blocked in the try_put(). */ +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + virtual graph_task* execute(const message_metainfo& metainfo) = 0; +#else + virtual graph_task* execute() = 0; +#endif + template friend class successor_cache; + bool is_continue_receiver() override { return true; } + + node_priority_t priority() const override { return my_priority; } +}; // class continue_receiver + +#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING + template + K key_from_message( const T &t ) { + return t.key(); + } +#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ + +} // d1 +} // detail +} // tbb + +#include "detail/_flow_graph_trace_impl.h" +#include "detail/_hash_compare.h" + +namespace tbb { +namespace detail { +namespace d2 { + +#include "detail/_flow_graph_body_impl.h" +#include "detail/_flow_graph_cache_impl.h" +#include "detail/_flow_graph_types_impl.h" + +using namespace graph_policy_namespace; + +template +graph_iterator::graph_iterator(C *g, bool begin) : my_graph(g), current_node(nullptr) +{ + if (begin) current_node = my_graph->my_nodes; + //else it is an end iterator by default +} + +template +typename graph_iterator::reference graph_iterator::operator*() const { + __TBB_ASSERT(current_node, "graph_iterator at end"); + return *operator->(); +} + +template +typename graph_iterator::pointer graph_iterator::operator->() const { + return current_node; +} + +template +void graph_iterator::internal_forward() { + if (current_node) current_node = current_node->next; +} + +//! Constructs a graph with isolated task_group_context +inline graph::graph() : my_wait_context_vertex(0), my_nodes(nullptr), my_nodes_last(nullptr), my_task_arena(nullptr) { + prepare_task_arena(); + own_context = true; + cancelled = false; + caught_exception = false; + my_context = new (r1::cache_aligned_allocate(sizeof(task_group_context))) task_group_context(FLOW_TASKS); + fgt_graph(this); + my_is_active = true; +} + +inline graph::graph(task_group_context& use_this_context) : + my_wait_context_vertex(0), my_context(&use_this_context), my_nodes(nullptr), my_nodes_last(nullptr), my_task_arena(nullptr) { + prepare_task_arena(); + own_context = false; + cancelled = false; + caught_exception = false; + fgt_graph(this); + my_is_active = true; +} + +inline graph::~graph() { + wait_for_all(); + if (own_context) { + my_context->~task_group_context(); + r1::cache_aligned_deallocate(my_context); + } + delete my_task_arena; +} + +inline void graph::reserve_wait() { + my_wait_context_vertex.reserve(); + fgt_reserve_wait(this); +} + +inline void graph::release_wait() { + fgt_release_wait(this); + my_wait_context_vertex.release(); +} + +inline void graph::register_node(graph_node *n) { + n->next = nullptr; + { + spin_mutex::scoped_lock lock(nodelist_mutex); + n->prev = my_nodes_last; + if (my_nodes_last) my_nodes_last->next = n; + my_nodes_last = n; + if (!my_nodes) my_nodes = n; + } +} + +inline void graph::remove_node(graph_node *n) { + { + spin_mutex::scoped_lock lock(nodelist_mutex); + __TBB_ASSERT(my_nodes && my_nodes_last, "graph::remove_node: Error: no registered nodes"); + if (n->prev) n->prev->next = n->next; + if (n->next) n->next->prev = n->prev; + if (my_nodes_last == n) my_nodes_last = n->prev; + if (my_nodes == n) my_nodes = n->next; + } + n->prev = n->next = nullptr; +} + +inline void graph::reset( reset_flags f ) { + // reset context + deactivate_graph(*this); + + my_context->reset(); + cancelled = false; + caught_exception = false; + // reset all the nodes comprising the graph + for(iterator ii = begin(); ii != end(); ++ii) { + graph_node *my_p = &(*ii); + my_p->reset_node(f); + } + // Reattach the arena. Might be useful to run the graph in a particular task_arena + // while not limiting graph lifetime to a single task_arena::execute() call. + prepare_task_arena( /*reinit=*/true ); + activate_graph(*this); +} + +inline void graph::cancel() { + my_context->cancel_group_execution(); +} + +inline graph::iterator graph::begin() { return iterator(this, true); } + +inline graph::iterator graph::end() { return iterator(this, false); } + +inline graph::const_iterator graph::begin() const { return const_iterator(this, true); } + +inline graph::const_iterator graph::end() const { return const_iterator(this, false); } + +inline graph::const_iterator graph::cbegin() const { return const_iterator(this, true); } + +inline graph::const_iterator graph::cend() const { return const_iterator(this, false); } + +inline graph_node::graph_node(graph& g) : my_graph(g) { + my_graph.register_node(this); +} + +inline graph_node::~graph_node() { + my_graph.remove_node(this); +} + +#include "detail/_flow_graph_node_impl.h" + + +//! An executable node that acts as a source, i.e. it has no predecessors + +template < typename Output > + __TBB_requires(std::copyable) +class input_node : public graph_node, public sender< Output > { +public: + //! The type of the output message, which is complete + typedef Output output_type; + + //! The type of successors of this node + typedef typename sender::successor_type successor_type; + + // Input node has no input type + typedef null_type input_type; + + //! Constructor for a node with a successor + template< typename Body > + __TBB_requires(input_node_body) + __TBB_NOINLINE_SYM input_node( graph &g, Body body ) + : graph_node(g), my_active(false) + , my_body( new input_body_leaf< output_type, Body>(body) ) + , my_init_body( new input_body_leaf< output_type, Body>(body) ) + , my_successors(this), my_reserved(false), my_has_cached_item(false) + { + fgt_node_with_body(CODEPTR(), FLOW_INPUT_NODE, &this->my_graph, + static_cast *>(this), this->my_body); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_requires(input_node_body) + input_node( const node_set& successors, Body body ) + : input_node(successors.graph_reference(), body) + { + make_edges(*this, successors); + } +#endif + + //! Copy constructor + __TBB_NOINLINE_SYM input_node( const input_node& src ) + : graph_node(src.my_graph), sender() + , my_active(false) + , my_body(src.my_init_body->clone()), my_init_body(src.my_init_body->clone()) + , my_successors(this), my_reserved(false), my_has_cached_item(false) + { + fgt_node_with_body(CODEPTR(), FLOW_INPUT_NODE, &this->my_graph, + static_cast *>(this), this->my_body); + } + + //! The destructor + ~input_node() { delete my_body; delete my_init_body; } + + //! Add a new successor to this node + bool register_successor( successor_type &r ) override { + spin_mutex::scoped_lock lock(my_mutex); + my_successors.register_successor(r); + if ( my_active ) + spawn_put(); + return true; + } + + //! Removes a successor from this node + bool remove_successor( successor_type &r ) override { + spin_mutex::scoped_lock lock(my_mutex); + my_successors.remove_successor(r); + return true; + } + + //! Request an item from the node + bool try_get( output_type &v ) override { + spin_mutex::scoped_lock lock(my_mutex); + if ( my_reserved ) + return false; + + if ( my_has_cached_item ) { + v = my_cached_item; + my_has_cached_item = false; + return true; + } + // we've been asked to provide an item, but we have none. enqueue a task to + // provide one. + if ( my_active ) + spawn_put(); + return false; + } + + //! Reserves an item. + bool try_reserve( output_type &v ) override { + spin_mutex::scoped_lock lock(my_mutex); + if ( my_reserved ) { + return false; + } + + if ( my_has_cached_item ) { + v = my_cached_item; + my_reserved = true; + return true; + } else { + return false; + } + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +private: + bool try_reserve( output_type& v, message_metainfo& ) override { + return try_reserve(v); + } + + bool try_get( output_type& v, message_metainfo& ) override { + return try_get(v); + } +public: +#endif + + //! Release a reserved item. + /** true = item has been released and so remains in sender, dest must request or reserve future items */ + bool try_release( ) override { + spin_mutex::scoped_lock lock(my_mutex); + __TBB_ASSERT( my_reserved && my_has_cached_item, "releasing non-existent reservation" ); + my_reserved = false; + if(!my_successors.empty()) + spawn_put(); + return true; + } + + //! Consumes a reserved item + bool try_consume( ) override { + spin_mutex::scoped_lock lock(my_mutex); + __TBB_ASSERT( my_reserved && my_has_cached_item, "consuming non-existent reservation" ); + my_reserved = false; + my_has_cached_item = false; + if ( !my_successors.empty() ) { + spawn_put(); + } + return true; + } + + //! Activates a node that was created in the inactive state + void activate() { + spin_mutex::scoped_lock lock(my_mutex); + my_active = true; + if (!my_successors.empty()) + spawn_put(); + } + + template + Body copy_function_object() { + input_body &body_ref = *this->my_body; + return dynamic_cast< input_body_leaf & >(body_ref).get_body(); + } + +protected: + + //! resets the input_node to its initial state + void reset_node( reset_flags f) override { + my_active = false; + my_reserved = false; + my_has_cached_item = false; + + if(f & rf_clear_edges) my_successors.clear(); + if(f & rf_reset_bodies) { + input_body *tmp = my_init_body->clone(); + delete my_body; + my_body = tmp; + } + } + +private: + spin_mutex my_mutex; + bool my_active; + input_body *my_body; + input_body *my_init_body; + broadcast_cache< output_type > my_successors; + bool my_reserved; + bool my_has_cached_item; + output_type my_cached_item; + + // used by apply_body_bypass, can invoke body of node. + bool try_reserve_apply_body(output_type &v) { + spin_mutex::scoped_lock lock(my_mutex); + if ( my_reserved ) { + return false; + } + if ( !my_has_cached_item ) { + d1::flow_control control; + + fgt_begin_body( my_body ); + + my_cached_item = (*my_body)(control); + my_has_cached_item = !control.is_pipeline_stopped; + + fgt_end_body( my_body ); + } + if ( my_has_cached_item ) { + v = my_cached_item; + my_reserved = true; + return true; + } else { + return false; + } + } + + graph_task* create_put_task() { + d1::small_object_allocator allocator{}; + typedef input_node_task_bypass< input_node > task_type; + graph_task* t = allocator.new_object(my_graph, allocator, *this); + return t; + } + + //! Spawns a task that applies the body + void spawn_put( ) { + if(is_graph_active(this->my_graph)) { + spawn_in_graph_arena(this->my_graph, *create_put_task()); + } + } + + friend class input_node_task_bypass< input_node >; + //! Applies the body. Returning SUCCESSFULLY_ENQUEUED okay; forward_task_bypass will handle it. + graph_task* apply_body_bypass( ) { + output_type v; + if ( !try_reserve_apply_body(v) ) + return nullptr; + + graph_task *last_task = my_successors.try_put_task(v); + if ( last_task ) + try_consume(); + else + try_release(); + return last_task; + } +}; // class input_node + +//! Implements a function node that supports Input -> Output +template + __TBB_requires(std::default_initializable && + std::copy_constructible && + std::copy_constructible) +class function_node + : public graph_node + , public function_input< Input, Output, Policy, cache_aligned_allocator > + , public function_output +{ + typedef cache_aligned_allocator internals_allocator; + +public: + typedef Input input_type; + typedef Output output_type; + typedef function_input input_impl_type; + typedef function_input_queue input_queue_type; + typedef function_output fOutput_type; + typedef typename input_impl_type::predecessor_type predecessor_type; + typedef typename fOutput_type::successor_type successor_type; + + using input_impl_type::my_predecessors; + + //! Constructor + // input_queue_type is allocated here, but destroyed in the function_input_base. + // TODO: pass the graph_buffer_policy to the function_input_base so it can all + // be done in one place. This would be an interface-breaking change. + template< typename Body > + __TBB_requires(function_node_body) + __TBB_NOINLINE_SYM function_node( graph &g, size_t concurrency, + Body body, Policy = Policy(), node_priority_t a_priority = no_priority ) + : graph_node(g), input_impl_type(g, concurrency, body, a_priority), + fOutput_type(g) { + fgt_node_with_body( CODEPTR(), FLOW_FUNCTION_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(this), this->my_body ); + } + + template + __TBB_requires(function_node_body) + function_node( graph& g, size_t concurrency, Body body, node_priority_t a_priority ) + : function_node(g, concurrency, body, Policy(), a_priority) {} + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_requires(function_node_body) + function_node( const node_set& nodes, size_t concurrency, Body body, + Policy p = Policy(), node_priority_t a_priority = no_priority ) + : function_node(nodes.graph_reference(), concurrency, body, p, a_priority) { + make_edges_in_order(nodes, *this); + } + + template + __TBB_requires(function_node_body) + function_node( const node_set& nodes, size_t concurrency, Body body, node_priority_t a_priority ) + : function_node(nodes, concurrency, body, Policy(), a_priority) {} +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + + //! Copy constructor + __TBB_NOINLINE_SYM function_node( const function_node& src ) : + graph_node(src.my_graph), + input_impl_type(src), + fOutput_type(src.my_graph) { + fgt_node_with_body( CODEPTR(), FLOW_FUNCTION_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(this), this->my_body ); + } + +protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + using input_impl_type::try_put_task; + + broadcast_cache &successors () override { return fOutput_type::my_successors; } + + void reset_node(reset_flags f) override { + input_impl_type::reset_function_input(f); + // TODO: use clear() instead. + if(f & rf_clear_edges) { + successors().clear(); + my_predecessors.clear(); + } + __TBB_ASSERT(!(f & rf_clear_edges) || successors().empty(), "function_node successors not empty"); + __TBB_ASSERT(this->my_predecessors.empty(), "function_node predecessors not empty"); + } + +}; // class function_node + +//! implements a function node that supports Input -> (set of outputs) +// Output is a tuple of output types. +template + __TBB_requires(std::default_initializable && + std::copy_constructible) +class multifunction_node : + public graph_node, + public multifunction_input + < + Input, + typename wrap_tuple_elements< + multifunction_output, // wrap this around each element + OutputTuple // Tuple with output types + >::type, + Policy, + cache_aligned_allocator + > +{ + typedef cache_aligned_allocator internals_allocator; + +protected: + static const int N = std::tuple_size::value; +public: + using input_type = Input; + using output_type = null_type; + using output_ports_type = typename wrap_tuple_elements::type; + using input_impl_type = + multifunction_input; + using input_queue_type = function_input_queue; +private: + using input_impl_type::my_predecessors; +public: + template + __TBB_requires(multifunction_node_body) + __TBB_NOINLINE_SYM multifunction_node( + graph &g, size_t concurrency, + Body body, Policy = Policy(), node_priority_t a_priority = no_priority + ) : graph_node(g), input_impl_type(g, concurrency, body, a_priority) { + fgt_multioutput_node_with_body( + CODEPTR(), FLOW_MULTIFUNCTION_NODE, + &this->my_graph, static_cast *>(this), + this->output_ports(), this->my_body + ); + } + + template + __TBB_requires(multifunction_node_body) + __TBB_NOINLINE_SYM multifunction_node(graph& g, size_t concurrency, Body body, node_priority_t a_priority) + : multifunction_node(g, concurrency, body, Policy(), a_priority) {} + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_requires(multifunction_node_body) + __TBB_NOINLINE_SYM multifunction_node(const node_set& nodes, size_t concurrency, Body body, + Policy p = Policy(), node_priority_t a_priority = no_priority) + : multifunction_node(nodes.graph_reference(), concurrency, body, p, a_priority) { + make_edges_in_order(nodes, *this); + } + + template + __TBB_requires(multifunction_node_body) + __TBB_NOINLINE_SYM multifunction_node(const node_set& nodes, size_t concurrency, Body body, node_priority_t a_priority) + : multifunction_node(nodes, concurrency, body, Policy(), a_priority) {} +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + + __TBB_NOINLINE_SYM multifunction_node( const multifunction_node &other) : + graph_node(other.my_graph), input_impl_type(other) { + fgt_multioutput_node_with_body( CODEPTR(), FLOW_MULTIFUNCTION_NODE, + &this->my_graph, static_cast *>(this), + this->output_ports(), this->my_body ); + } + + // all the guts are in multifunction_input... +protected: + void reset_node(reset_flags f) override { input_impl_type::reset(f); } +}; // multifunction_node + +//! split_node: accepts a tuple as input, forwards each element of the tuple to its +// successors. The node has unlimited concurrency, so it does not reject inputs. +template +class split_node : public graph_node, public receiver { + static const int N = std::tuple_size::value; + typedef receiver base_type; +public: + typedef TupleType input_type; + typedef typename wrap_tuple_elements< + multifunction_output, // wrap this around each element + TupleType // the tuple providing the types + >::type output_ports_type; + + __TBB_NOINLINE_SYM explicit split_node(graph &g) + : graph_node(g), + my_output_ports(init_output_ports::call(g, my_output_ports)) + { + fgt_multioutput_node(CODEPTR(), FLOW_SPLIT_NODE, &this->my_graph, + static_cast *>(this), this->output_ports()); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_NOINLINE_SYM split_node(const node_set& nodes) : split_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + __TBB_NOINLINE_SYM split_node(const split_node& other) + : graph_node(other.my_graph), base_type(other), + my_output_ports(init_output_ports::call(other.my_graph, my_output_ports)) + { + fgt_multioutput_node(CODEPTR(), FLOW_SPLIT_NODE, &this->my_graph, + static_cast *>(this), this->output_ports()); + } + + output_ports_type &output_ports() { return my_output_ports; } + +protected: + graph_task *try_put_task(const TupleType& t) override { + // Sending split messages in parallel is not justified, as overheads would prevail. + // Also, we do not have successors here. So we just tell the task returned here is successful. + return emit_element::emit_this(this->my_graph, t, output_ports()); + } +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const TupleType& t, const message_metainfo& metainfo) override { + // Sending split messages in parallel is not justified, as overheads would prevail. + // Also, we do not have successors here. So we just tell the task returned here is successful. + return emit_element::emit_this(this->my_graph, t, output_ports(), metainfo); + } +#endif + + void reset_node(reset_flags f) override { + if (f & rf_clear_edges) + clear_element::clear_this(my_output_ports); + + __TBB_ASSERT(!(f & rf_clear_edges) || clear_element::this_empty(my_output_ports), "split_node reset failed"); + } + graph& graph_reference() const override { + return my_graph; + } + +private: + output_ports_type my_output_ports; +}; + +//! Implements an executable node that supports continue_msg -> Output +template > + __TBB_requires(std::copy_constructible) +class continue_node : public graph_node, public continue_input, + public function_output { +public: + typedef continue_msg input_type; + typedef Output output_type; + typedef continue_input input_impl_type; + typedef function_output fOutput_type; + typedef typename input_impl_type::predecessor_type predecessor_type; + typedef typename fOutput_type::successor_type successor_type; + + //! Constructor for executable node with continue_msg -> Output + template + __TBB_requires(continue_node_body) + __TBB_NOINLINE_SYM continue_node( + graph &g, + Body body, Policy = Policy(), node_priority_t a_priority = no_priority + ) : graph_node(g), input_impl_type( g, body, a_priority ), + fOutput_type(g) { + fgt_node_with_body( CODEPTR(), FLOW_CONTINUE_NODE, &this->my_graph, + + static_cast *>(this), + static_cast *>(this), this->my_body ); + } + + template + __TBB_requires(continue_node_body) + continue_node( graph& g, Body body, node_priority_t a_priority ) + : continue_node(g, body, Policy(), a_priority) {} + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_requires(continue_node_body) + continue_node( const node_set& nodes, Body body, + Policy p = Policy(), node_priority_t a_priority = no_priority ) + : continue_node(nodes.graph_reference(), body, p, a_priority ) { + make_edges_in_order(nodes, *this); + } + template + __TBB_requires(continue_node_body) + continue_node( const node_set& nodes, Body body, node_priority_t a_priority) + : continue_node(nodes, body, Policy(), a_priority) {} +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + + //! Constructor for executable node with continue_msg -> Output + template + __TBB_requires(continue_node_body) + __TBB_NOINLINE_SYM continue_node( + graph &g, int number_of_predecessors, + Body body, Policy = Policy(), node_priority_t a_priority = no_priority + ) : graph_node(g) + , input_impl_type(g, number_of_predecessors, body, a_priority), + fOutput_type(g) { + fgt_node_with_body( CODEPTR(), FLOW_CONTINUE_NODE, &this->my_graph, + static_cast *>(this), + static_cast *>(this), this->my_body ); + } + + template + __TBB_requires(continue_node_body) + continue_node( graph& g, int number_of_predecessors, Body body, node_priority_t a_priority) + : continue_node(g, number_of_predecessors, body, Policy(), a_priority) {} + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_requires(continue_node_body) + continue_node( const node_set& nodes, int number_of_predecessors, + Body body, Policy p = Policy(), node_priority_t a_priority = no_priority ) + : continue_node(nodes.graph_reference(), number_of_predecessors, body, p, a_priority) { + make_edges_in_order(nodes, *this); + } + + template + __TBB_requires(continue_node_body) + continue_node( const node_set& nodes, int number_of_predecessors, + Body body, node_priority_t a_priority ) + : continue_node(nodes, number_of_predecessors, body, Policy(), a_priority) {} +#endif + + //! Copy constructor + __TBB_NOINLINE_SYM continue_node( const continue_node& src ) : + graph_node(src.my_graph), input_impl_type(src), + function_output(src.my_graph) { + fgt_node_with_body( CODEPTR(), FLOW_CONTINUE_NODE, &this->my_graph, + static_cast *>(this), + static_cast *>(this), this->my_body ); + } + +protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + using input_impl_type::try_put_task; + broadcast_cache &successors () override { return fOutput_type::my_successors; } + + void reset_node(reset_flags f) override { + input_impl_type::reset_receiver(f); + if(f & rf_clear_edges)successors().clear(); + __TBB_ASSERT(!(f & rf_clear_edges) || successors().empty(), "continue_node not reset"); + } +}; // continue_node + +//! Forwards messages of type T to all successors +template +class broadcast_node : public graph_node, public receiver, public sender { +public: + typedef T input_type; + typedef T output_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; +private: + broadcast_cache my_successors; +public: + + __TBB_NOINLINE_SYM explicit broadcast_node(graph& g) : graph_node(g), my_successors(this) { + fgt_node( CODEPTR(), FLOW_BROADCAST_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + broadcast_node(const node_set& nodes) : broadcast_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + // Copy constructor + __TBB_NOINLINE_SYM broadcast_node( const broadcast_node& src ) : broadcast_node(src.my_graph) {} + + //! Adds a successor + bool register_successor( successor_type &r ) override { + my_successors.register_successor( r ); + return true; + } + + //! Removes s as a successor + bool remove_successor( successor_type &r ) override { + my_successors.remove_successor( r ); + return true; + } + +private: + graph_task* try_put_task_impl(const T& t __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) { + graph_task* new_task = my_successors.try_put_task(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + if (!new_task) new_task = SUCCESSFULLY_ENQUEUED; + return new_task; + } + +protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + //! build a task to run the successor if possible. Default is old behavior. + graph_task* try_put_task(const T& t) override { + return try_put_task_impl(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const T& t, const message_metainfo& metainfo) override { + return try_put_task_impl(t, metainfo); + } +#endif + + graph& graph_reference() const override { + return my_graph; + } + + void reset_node(reset_flags f) override { + if (f&rf_clear_edges) { + my_successors.clear(); + } + __TBB_ASSERT(!(f & rf_clear_edges) || my_successors.empty(), "Error resetting broadcast_node"); + } +}; // broadcast_node + +//! Forwards messages in arbitrary order +template +class buffer_node + : public graph_node + , public reservable_item_buffer< T, cache_aligned_allocator > + , public receiver, public sender +{ + typedef cache_aligned_allocator internals_allocator; + +public: + typedef T input_type; + typedef T output_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; + typedef buffer_node class_type; + +protected: + typedef size_t size_type; + round_robin_cache< T, null_rw_mutex > my_successors; + + friend class forward_task_bypass< class_type >; + + enum op_type {reg_succ, rem_succ, req_item, res_item, rel_res, con_res, put_item, try_fwd_task + }; + + // implements the aggregator_operation concept + class buffer_operation : public d1::aggregated_operation< buffer_operation > { + public: + char type; + T* elem; + graph_task* ltask; + successor_type *r; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo* metainfo{ nullptr }; +#endif + + buffer_operation(const T& e, op_type t) : type(char(t)) + , elem(const_cast(&e)) , ltask(nullptr) + , r(nullptr) + {} + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + buffer_operation(const T& e, op_type t, const message_metainfo& info) + : type(char(t)), elem(const_cast(&e)), ltask(nullptr), r(nullptr) + , metainfo(const_cast(&info)) + {} + + buffer_operation(op_type t, message_metainfo& info) + : type(char(t)), elem(nullptr), ltask(nullptr), r(nullptr), metainfo(&info) {} +#endif + buffer_operation(op_type t) : type(char(t)), elem(nullptr), ltask(nullptr), r(nullptr) {} + }; + + bool forwarder_busy; + typedef d1::aggregating_functor handler_type; + friend class d1::aggregating_functor; + d1::aggregator< handler_type, buffer_operation> my_aggregator; + + virtual void handle_operations(buffer_operation *op_list) { + handle_operations_impl(op_list, this); + } + + template + void handle_operations_impl(buffer_operation *op_list, derived_type* derived) { + __TBB_ASSERT(static_cast(derived) == this, "'this' is not a base class for derived"); + + buffer_operation *tmp = nullptr; + bool try_forwarding = false; + while (op_list) { + tmp = op_list; + op_list = op_list->next; + switch (tmp->type) { + case reg_succ: internal_reg_succ(tmp); try_forwarding = true; break; + case rem_succ: internal_rem_succ(tmp); break; + case req_item: internal_pop(tmp); break; + case res_item: internal_reserve(tmp); break; + case rel_res: internal_release(tmp); try_forwarding = true; break; + case con_res: internal_consume(tmp); try_forwarding = true; break; + case put_item: try_forwarding = internal_push(tmp); break; + case try_fwd_task: internal_forward_task(tmp); break; + } + } + + derived->order(); + + if (try_forwarding && !forwarder_busy) { + if(is_graph_active(this->my_graph)) { + forwarder_busy = true; + typedef forward_task_bypass task_type; + d1::small_object_allocator allocator{}; + graph_task* new_task = allocator.new_object(graph_reference(), allocator, *this); + // tmp should point to the last item handled by the aggregator. This is the operation + // the handling thread enqueued. So modifying that record will be okay. + // TODO revamp: check that the issue is still present + // workaround for icc bug (at least 12.0 and 13.0) + // error: function "tbb::flow::interfaceX::combine_tasks" cannot be called with the given argument list + // argument types are: (graph, graph_task *, graph_task *) + graph_task *z = tmp->ltask; + graph &g = this->my_graph; + tmp->ltask = combine_tasks(g, z, new_task); // in case the op generated a task + } + } + } // handle_operations + + inline graph_task *grab_forwarding_task( buffer_operation &op_data) { + return op_data.ltask; + } + + inline bool enqueue_forwarding_task(buffer_operation &op_data) { + graph_task *ft = grab_forwarding_task(op_data); + if(ft) { + spawn_in_graph_arena(graph_reference(), *ft); + return true; + } + return false; + } + + //! This is executed by an enqueued task, the "forwarder" + virtual graph_task *forward_task() { + buffer_operation op_data(try_fwd_task); + graph_task *last_task = nullptr; + do { + op_data.status = WAIT; + op_data.ltask = nullptr; + my_aggregator.execute(&op_data); + + // workaround for icc bug + graph_task *xtask = op_data.ltask; + graph& g = this->my_graph; + last_task = combine_tasks(g, last_task, xtask); + } while (op_data.status ==SUCCEEDED); + return last_task; + } + + //! Register successor + virtual void internal_reg_succ(buffer_operation *op) { + __TBB_ASSERT(op->r, nullptr); + my_successors.register_successor(*(op->r)); + op->status.store(SUCCEEDED, std::memory_order_release); + } + + //! Remove successor + virtual void internal_rem_succ(buffer_operation *op) { + __TBB_ASSERT(op->r, nullptr); + my_successors.remove_successor(*(op->r)); + op->status.store(SUCCEEDED, std::memory_order_release); + } + +private: + void order() {} + + bool is_item_valid() { + return this->my_item_valid(this->my_tail - 1); + } + + void try_put_and_add_task(graph_task*& last_task) { + graph_task* new_task = my_successors.try_put_task(this->back() + __TBB_FLOW_GRAPH_METAINFO_ARG(this->back_metainfo())); + if (new_task) { + // workaround for icc bug + graph& g = this->my_graph; + last_task = combine_tasks(g, last_task, new_task); + this->destroy_back(); + } + } + +protected: + //! Tries to forward valid items to successors + virtual void internal_forward_task(buffer_operation *op) { + internal_forward_task_impl(op, this); + } + + template + void internal_forward_task_impl(buffer_operation *op, derived_type* derived) { + __TBB_ASSERT(static_cast(derived) == this, "'this' is not a base class for derived"); + + if (this->my_reserved || !derived->is_item_valid()) { + op->status.store(FAILED, std::memory_order_release); + this->forwarder_busy = false; + return; + } + // Try forwarding, giving each successor a chance + graph_task* last_task = nullptr; + size_type counter = my_successors.size(); + for (; counter > 0 && derived->is_item_valid(); --counter) + derived->try_put_and_add_task(last_task); + + op->ltask = last_task; // return task + if (last_task && !counter) { + op->status.store(SUCCEEDED, std::memory_order_release); + } + else { + op->status.store(FAILED, std::memory_order_release); + forwarder_busy = false; + } + } + + virtual bool internal_push(buffer_operation *op) { + __TBB_ASSERT(op->elem, nullptr); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + __TBB_ASSERT(op->metainfo, nullptr); + this->push_back(*(op->elem), (*op->metainfo)); +#else + this->push_back(*(op->elem)); +#endif + op->status.store(SUCCEEDED, std::memory_order_release); + return true; + } + + virtual void internal_pop(buffer_operation *op) { + __TBB_ASSERT(op->elem, nullptr); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool pop_result = op->metainfo ? this->pop_back(*(op->elem), *(op->metainfo)) + : this->pop_back(*(op->elem)); +#else + bool pop_result = this->pop_back(*(op->elem)); +#endif + if (pop_result) { + op->status.store(SUCCEEDED, std::memory_order_release); + } + else { + op->status.store(FAILED, std::memory_order_release); + } + } + + virtual void internal_reserve(buffer_operation *op) { + __TBB_ASSERT(op->elem, nullptr); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool reserve_result = op->metainfo ? this->reserve_front(*(op->elem), *(op->metainfo)) + : this->reserve_front(*(op->elem)); +#else + bool reserve_result = this->reserve_front(*(op->elem)); +#endif + if (reserve_result) { + op->status.store(SUCCEEDED, std::memory_order_release); + } + else { + op->status.store(FAILED, std::memory_order_release); + } + } + + virtual void internal_consume(buffer_operation *op) { + this->consume_front(); + op->status.store(SUCCEEDED, std::memory_order_release); + } + + virtual void internal_release(buffer_operation *op) { + this->release_front(); + op->status.store(SUCCEEDED, std::memory_order_release); + } + +public: + //! Constructor + __TBB_NOINLINE_SYM explicit buffer_node( graph &g ) + : graph_node(g), reservable_item_buffer(), receiver(), + sender(), my_successors(this), forwarder_busy(false) + { + my_aggregator.initialize_handler(handler_type(this)); + fgt_node( CODEPTR(), FLOW_BUFFER_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + buffer_node(const node_set& nodes) : buffer_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + //! Copy constructor + __TBB_NOINLINE_SYM buffer_node( const buffer_node& src ) : buffer_node(src.my_graph) {} + + // + // message sender implementation + // + + //! Adds a new successor. + /** Adds successor r to the list of successors; may forward tasks. */ + bool register_successor( successor_type &r ) override { + buffer_operation op_data(reg_succ); + op_data.r = &r; + my_aggregator.execute(&op_data); + (void)enqueue_forwarding_task(op_data); + return true; + } + + //! Removes a successor. + /** Removes successor r from the list of successors. + It also calls r.remove_predecessor(*this) to remove this node as a predecessor. */ + bool remove_successor( successor_type &r ) override { + // TODO revamp: investigate why full qualification is necessary here + tbb::detail::d2::remove_predecessor(r, *this); + buffer_operation op_data(rem_succ); + op_data.r = &r; + my_aggregator.execute(&op_data); + // even though this operation does not cause a forward, if we are the handler, and + // a forward is scheduled, we may be the first to reach this point after the aggregator, + // and so should check for the task. + (void)enqueue_forwarding_task(op_data); + return true; + } + + //! Request an item from the buffer_node + /** true = v contains the returned item
+ false = no item has been returned */ + bool try_get( T &v ) override { + buffer_operation op_data(req_item); + op_data.elem = &v; + my_aggregator.execute(&op_data); + (void)enqueue_forwarding_task(op_data); + return (op_data.status==SUCCEEDED); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool try_get( T &v, message_metainfo& metainfo ) override { + buffer_operation op_data(req_item, metainfo); + op_data.elem = &v; + my_aggregator.execute(&op_data); + (void)enqueue_forwarding_task(op_data); + return (op_data.status==SUCCEEDED); + } +#endif + + //! Reserves an item. + /** false = no item can be reserved
+ true = an item is reserved */ + bool try_reserve( T &v ) override { + buffer_operation op_data(res_item); + op_data.elem = &v; + my_aggregator.execute(&op_data); + (void)enqueue_forwarding_task(op_data); + return (op_data.status==SUCCEEDED); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool try_reserve( output_type& v, message_metainfo& metainfo ) override { + buffer_operation op_data(res_item, metainfo); + op_data.elem = &v; + my_aggregator.execute(&op_data); + (void)enqueue_forwarding_task(op_data); + return op_data.status==SUCCEEDED; + } +#endif + + //! Release a reserved item. + /** true = item has been released and so remains in sender */ + bool try_release() override { + buffer_operation op_data(rel_res); + my_aggregator.execute(&op_data); + (void)enqueue_forwarding_task(op_data); + return true; + } + + //! Consumes a reserved item. + /** true = item is removed from sender and reservation removed */ + bool try_consume() override { + buffer_operation op_data(con_res); + my_aggregator.execute(&op_data); + (void)enqueue_forwarding_task(op_data); + return true; + } + +private: + graph_task* try_put_task_impl(const T& t __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) { + buffer_operation op_data(t, put_item __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + my_aggregator.execute(&op_data); + graph_task *ft = grab_forwarding_task(op_data); + // sequencer_nodes can return failure (if an item has been previously inserted) + // We have to spawn the returned task if our own operation fails. + + if(ft && op_data.status ==FAILED) { + // we haven't succeeded queueing the item, but for some reason the + // call returned a task (if another request resulted in a successful + // forward this could happen.) Queue the task and reset the pointer. + spawn_in_graph_arena(graph_reference(), *ft); ft = nullptr; + } + else if(!ft && op_data.status ==SUCCEEDED) { + ft = SUCCESSFULLY_ENQUEUED; + } + return ft; + } + +protected: + + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + //! receive an item, return a task *if possible + graph_task *try_put_task(const T &t) override { + return try_put_task_impl(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const T& t, const message_metainfo& metainfo) override { + return try_put_task_impl(t, metainfo); + } +#endif + + graph& graph_reference() const override { + return my_graph; + } + +protected: + void reset_node( reset_flags f) override { + reservable_item_buffer::reset(); + // TODO: just clear structures + if (f&rf_clear_edges) { + my_successors.clear(); + } + forwarder_busy = false; + } +}; // buffer_node + +//! Forwards messages in FIFO order +template +class queue_node : public buffer_node { +protected: + typedef buffer_node base_type; + typedef typename base_type::size_type size_type; + typedef typename base_type::buffer_operation queue_operation; + typedef queue_node class_type; + +private: + template friend class buffer_node; + + bool is_item_valid() { + return this->my_item_valid(this->my_head); + } + + void try_put_and_add_task(graph_task*& last_task) { + graph_task* new_task = this->my_successors.try_put_task(this->front() + __TBB_FLOW_GRAPH_METAINFO_ARG(this->front_metainfo())); + + if (new_task) { + // workaround for icc bug + graph& graph_ref = this->graph_reference(); + last_task = combine_tasks(graph_ref, last_task, new_task); + this->destroy_front(); + } + } + +protected: + void internal_forward_task(queue_operation *op) override { + this->internal_forward_task_impl(op, this); + } + + void internal_pop(queue_operation *op) override { + if ( this->my_reserved || !this->my_item_valid(this->my_head)){ + op->status.store(FAILED, std::memory_order_release); + } + else { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (op->metainfo) { + this->pop_front(*(op->elem), *(op->metainfo)); + } else +#endif + { + this->pop_front(*(op->elem)); + } + op->status.store(SUCCEEDED, std::memory_order_release); + } + } + void internal_reserve(queue_operation *op) override { + if (this->my_reserved || !this->my_item_valid(this->my_head)) { + op->status.store(FAILED, std::memory_order_release); + } + else { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (op->metainfo) { + this->reserve_front(*(op->elem), *(op->metainfo)); + } + else +#endif + { + this->reserve_front(*(op->elem)); + } + op->status.store(SUCCEEDED, std::memory_order_release); + } + } + void internal_consume(queue_operation *op) override { + this->consume_front(); + op->status.store(SUCCEEDED, std::memory_order_release); + } + +public: + typedef T input_type; + typedef T output_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; + + //! Constructor + __TBB_NOINLINE_SYM explicit queue_node( graph &g ) : base_type(g) { + fgt_node( CODEPTR(), FLOW_QUEUE_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + queue_node( const node_set& nodes) : queue_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + //! Copy constructor + __TBB_NOINLINE_SYM queue_node( const queue_node& src) : base_type(src) { + fgt_node( CODEPTR(), FLOW_QUEUE_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + + +protected: + void reset_node( reset_flags f) override { + base_type::reset_node(f); + } +}; // queue_node + +//! Forwards messages in sequence order +template + __TBB_requires(std::copyable) +class sequencer_node : public queue_node { + function_body< T, size_t > *my_sequencer; + // my_sequencer should be a benign function and must be callable + // from a parallel context. Does this mean it needn't be reset? +public: + typedef T input_type; + typedef T output_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; + + //! Constructor + template< typename Sequencer > + __TBB_requires(sequencer) + __TBB_NOINLINE_SYM sequencer_node( graph &g, const Sequencer& s ) : queue_node(g), + my_sequencer(new function_body_leaf< T, size_t, Sequencer>(s) ) { + fgt_node( CODEPTR(), FLOW_SEQUENCER_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_requires(sequencer) + sequencer_node( const node_set& nodes, const Sequencer& s) + : sequencer_node(nodes.graph_reference(), s) { + make_edges_in_order(nodes, *this); + } +#endif + + //! Copy constructor + __TBB_NOINLINE_SYM sequencer_node( const sequencer_node& src ) : queue_node(src), + my_sequencer( src.my_sequencer->clone() ) { + fgt_node( CODEPTR(), FLOW_SEQUENCER_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + + //! Destructor + ~sequencer_node() { delete my_sequencer; } + +protected: + typedef typename buffer_node::size_type size_type; + typedef typename buffer_node::buffer_operation sequencer_operation; + +private: + bool internal_push(sequencer_operation *op) override { + size_type tag = (*my_sequencer)(*(op->elem)); +#if !TBB_DEPRECATED_SEQUENCER_DUPLICATES + if (tag < this->my_head) { + // have already emitted a message with this tag + op->status.store(FAILED, std::memory_order_release); + return false; + } +#endif + // cannot modify this->my_tail now; the buffer would be inconsistent. + size_t new_tail = (tag+1 > this->my_tail) ? tag+1 : this->my_tail; + + if (this->size(new_tail) > this->capacity()) { + this->grow_my_array(this->size(new_tail)); + } + this->my_tail = new_tail; + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + __TBB_ASSERT(op->metainfo, nullptr); + bool place_item_result = this->place_item(tag, *(op->elem), *(op->metainfo)); + const op_stat res = place_item_result ? SUCCEEDED : FAILED; +#else + const op_stat res = this->place_item(tag, *(op->elem)) ? SUCCEEDED : FAILED; +#endif + op->status.store(res, std::memory_order_release); + return res ==SUCCEEDED; + } +}; // sequencer_node + +//! Forwards messages in priority order +template> +class priority_queue_node : public buffer_node { +public: + typedef T input_type; + typedef T output_type; + typedef buffer_node base_type; + typedef priority_queue_node class_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; + + //! Constructor + __TBB_NOINLINE_SYM explicit priority_queue_node( graph &g, const Compare& comp = Compare() ) + : buffer_node(g), compare(comp), mark(0) { + fgt_node( CODEPTR(), FLOW_PRIORITY_QUEUE_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + priority_queue_node(const node_set& nodes, const Compare& comp = Compare()) + : priority_queue_node(nodes.graph_reference(), comp) { + make_edges_in_order(nodes, *this); + } +#endif + + //! Copy constructor + __TBB_NOINLINE_SYM priority_queue_node( const priority_queue_node &src ) + : buffer_node(src), mark(0) + { + fgt_node( CODEPTR(), FLOW_PRIORITY_QUEUE_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + +protected: + + void reset_node( reset_flags f) override { + mark = 0; + base_type::reset_node(f); + } + + typedef typename buffer_node::size_type size_type; + typedef typename buffer_node::item_type item_type; + typedef typename buffer_node::buffer_operation prio_operation; + + //! Tries to forward valid items to successors + void internal_forward_task(prio_operation *op) override { + this->internal_forward_task_impl(op, this); + } + + void handle_operations(prio_operation *op_list) override { + this->handle_operations_impl(op_list, this); + } + + bool internal_push(prio_operation *op) override { +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + __TBB_ASSERT(op->metainfo, nullptr); + prio_push(*(op->elem), *(op->metainfo)); +#else + prio_push(*(op->elem)); +#endif + op->status.store(SUCCEEDED, std::memory_order_release); + return true; + } + + void internal_pop(prio_operation *op) override { + // if empty or already reserved, don't pop + if ( this->my_reserved == true || this->my_tail == 0 ) { + op->status.store(FAILED, std::memory_order_release); + return; + } + + *(op->elem) = prio(); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (op->metainfo) { + *(op->metainfo) = std::move(prio_metainfo()); + } +#endif + op->status.store(SUCCEEDED, std::memory_order_release); + prio_pop(); + + } + + // pops the highest-priority item, saves copy + void internal_reserve(prio_operation *op) override { + if (this->my_reserved == true || this->my_tail == 0) { + op->status.store(FAILED, std::memory_order_release); + return; + } + this->my_reserved = true; + *(op->elem) = prio(); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + if (op->metainfo) { + *(op->metainfo) = std::move(prio_metainfo()); + reserved_metainfo = *(op->metainfo); + } +#endif + reserved_item = *(op->elem); + op->status.store(SUCCEEDED, std::memory_order_release); + prio_pop(); + } + + void internal_consume(prio_operation *op) override { + op->status.store(SUCCEEDED, std::memory_order_release); + this->my_reserved = false; + reserved_item = input_type(); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + for (auto waiter : reserved_metainfo.waiters()) { + waiter->release(1); + } + + reserved_metainfo = message_metainfo{}; +#endif + } + + void internal_release(prio_operation *op) override { + op->status.store(SUCCEEDED, std::memory_order_release); + prio_push(reserved_item __TBB_FLOW_GRAPH_METAINFO_ARG(reserved_metainfo)); + this->my_reserved = false; + reserved_item = input_type(); +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + for (auto waiter : reserved_metainfo.waiters()) { + waiter->release(1); + } + + reserved_metainfo = message_metainfo{}; +#endif + } + +private: + template friend class buffer_node; + + void order() { + if (mark < this->my_tail) heapify(); + __TBB_ASSERT(mark == this->my_tail, "mark unequal after heapify"); + } + + bool is_item_valid() { + return this->my_tail > 0; + } + + void try_put_and_add_task(graph_task*& last_task) { + graph_task* new_task = this->my_successors.try_put_task(this->prio() + __TBB_FLOW_GRAPH_METAINFO_ARG(this->prio_metainfo())); + if (new_task) { + // workaround for icc bug + graph& graph_ref = this->graph_reference(); + last_task = combine_tasks(graph_ref, last_task, new_task); + prio_pop(); + } + } + +private: + Compare compare; + size_type mark; + + input_type reserved_item; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo reserved_metainfo; +#endif + + // in case a reheap has not been done after a push, check if the mark item is higher than the 0'th item + bool prio_use_tail() { + __TBB_ASSERT(mark <= this->my_tail, "mark outside bounds before test"); + return mark < this->my_tail && compare(this->get_my_item(0), this->get_my_item(this->my_tail - 1)); + } + + // prio_push: checks that the item will fit, expand array if necessary, put at end + void prio_push(const T &src __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) { + if ( this->my_tail >= this->my_array_size ) + this->grow_my_array( this->my_tail + 1 ); + (void) this->place_item(this->my_tail, src __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + ++(this->my_tail); + __TBB_ASSERT(mark < this->my_tail, "mark outside bounds after push"); + } + + // prio_pop: deletes highest priority item from the array, and if it is item + // 0, move last item to 0 and reheap. If end of array, just destroy and decrement tail + // and mark. Assumes the array has already been tested for emptiness; no failure. + void prio_pop() { + if (prio_use_tail()) { + // there are newly pushed elements; last one higher than top + // copy the data + this->destroy_item(this->my_tail-1); + --(this->my_tail); + __TBB_ASSERT(mark <= this->my_tail, "mark outside bounds after pop"); + return; + } + this->destroy_item(0); + if(this->my_tail > 1) { + // push the last element down heap + __TBB_ASSERT(this->my_item_valid(this->my_tail - 1), nullptr); + this->move_item(0,this->my_tail - 1); + } + --(this->my_tail); + if(mark > this->my_tail) --mark; + if (this->my_tail > 1) // don't reheap for heap of size 1 + reheap(); + __TBB_ASSERT(mark <= this->my_tail, "mark outside bounds after pop"); + } + + const T& prio() { + return this->get_my_item(prio_use_tail() ? this->my_tail-1 : 0); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo& prio_metainfo() { + return this->get_my_metainfo(prio_use_tail() ? this->my_tail-1 : 0); + } +#endif + + // turn array into heap + void heapify() { + if(this->my_tail == 0) { + mark = 0; + return; + } + if (!mark) mark = 1; + for (; markmy_tail; ++mark) { // for each unheaped element + size_type cur_pos = mark; + input_type to_place; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo metainfo; +#endif + this->fetch_item(mark, to_place __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + do { // push to_place up the heap + size_type parent = (cur_pos-1)>>1; + if (!compare(this->get_my_item(parent), to_place)) + break; + this->move_item(cur_pos, parent); + cur_pos = parent; + } while( cur_pos ); + this->place_item(cur_pos, to_place __TBB_FLOW_GRAPH_METAINFO_ARG(std::move(metainfo))); + } + } + + // otherwise heapified array with new root element; rearrange to heap + void reheap() { + size_type cur_pos=0, child=1; + while (child < mark) { + size_type target = child; + if (child+1get_my_item(child), + this->get_my_item(child+1))) + ++target; + // target now has the higher priority child + if (compare(this->get_my_item(target), + this->get_my_item(cur_pos))) + break; + // swap + this->swap_items(cur_pos, target); + cur_pos = target; + child = (cur_pos<<1)+1; + } + } +}; // priority_queue_node + +//! Forwards messages only if the threshold has not been reached +/** This node forwards items until its threshold is reached. + It contains no buffering. If the downstream node rejects, the + message is dropped. */ +template< typename T, typename DecrementType=continue_msg > +class limiter_node : public graph_node, public receiver< T >, public sender< T > { +public: + typedef T input_type; + typedef T output_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; + //TODO: There is a lack of predefined types for its controlling "decrementer" port. It should be fixed later. + +private: + size_t my_threshold; + size_t my_count; // number of successful puts + size_t my_tries; // number of active put attempts + size_t my_future_decrement; // number of active decrement + reservable_predecessor_cache< T, spin_mutex > my_predecessors; + spin_mutex my_mutex; + broadcast_cache< T > my_successors; + + //! The internal receiver< DecrementType > that adjusts the count + threshold_regulator< limiter_node, DecrementType > decrement; + + graph_task* decrement_counter( long long delta ) { + if ( delta > 0 && size_t(delta) > my_threshold ) { + delta = my_threshold; + } + + { + spin_mutex::scoped_lock lock(my_mutex); + if ( delta > 0 && size_t(delta) > my_count ) { + if( my_tries > 0 ) { + my_future_decrement += (size_t(delta) - my_count); + } + my_count = 0; + } + else if ( delta < 0 && size_t(-delta) > my_threshold - my_count ) { + my_count = my_threshold; + } + else { + my_count -= size_t(delta); // absolute value of delta is sufficiently small + } + __TBB_ASSERT(my_count <= my_threshold, "counter values are truncated to be inside the [0, threshold] interval"); + } + return forward_task(); + } + + // Let threshold_regulator call decrement_counter() + friend class threshold_regulator< limiter_node, DecrementType >; + + friend class forward_task_bypass< limiter_node >; + + bool check_conditions() { // always called under lock + return ( my_count + my_tries < my_threshold && !my_predecessors.empty() && !my_successors.empty() ); + } + + // only returns a valid task pointer or nullptr, never SUCCESSFULLY_ENQUEUED + graph_task* forward_task() { + input_type v; + graph_task* rval = nullptr; + bool reserved = false; + + { + spin_mutex::scoped_lock lock(my_mutex); + if ( check_conditions() ) + ++my_tries; + else + return nullptr; + } + + //SUCCESS + // if we can reserve and can put, we consume the reservation + // we increment the count and decrement the tries +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo metainfo; +#endif + if ( (my_predecessors.try_reserve(v __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo))) == true ) { + reserved = true; + if ( (rval = my_successors.try_put_task(v __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo))) != nullptr ) { + { + spin_mutex::scoped_lock lock(my_mutex); + ++my_count; + if ( my_future_decrement ) { + if ( my_count > my_future_decrement ) { + my_count -= my_future_decrement; + my_future_decrement = 0; + } + else { + my_future_decrement -= my_count; + my_count = 0; + } + } + --my_tries; + my_predecessors.try_consume(); + if ( check_conditions() ) { + if ( is_graph_active(this->my_graph) ) { + typedef forward_task_bypass> task_type; + d1::small_object_allocator allocator{}; + graph_task* rtask = allocator.new_object( my_graph, allocator, *this ); + spawn_in_graph_arena(graph_reference(), *rtask); + } + } + } + return rval; + } + } + //FAILURE + //if we can't reserve, we decrement the tries + //if we can reserve but can't put, we decrement the tries and release the reservation + { + spin_mutex::scoped_lock lock(my_mutex); + --my_tries; + if (reserved) my_predecessors.try_release(); + if ( check_conditions() ) { + if ( is_graph_active(this->my_graph) ) { + d1::small_object_allocator allocator{}; + typedef forward_task_bypass> task_type; + graph_task* t = allocator.new_object(my_graph, allocator, *this); + __TBB_ASSERT(!rval, "Have two tasks to handle"); + return t; + } + } + return rval; + } + } + + void initialize() { + fgt_node( + CODEPTR(), FLOW_LIMITER_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(&decrement), + static_cast *>(this) + ); + } + +public: + //! Constructor + limiter_node(graph &g, size_t threshold) + : graph_node(g), my_threshold(threshold), my_count(0), my_tries(0), my_future_decrement(0), + my_predecessors(this), my_successors(this), decrement(this) + { + initialize(); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + limiter_node(const node_set& nodes, size_t threshold) + : limiter_node(nodes.graph_reference(), threshold) { + make_edges_in_order(nodes, *this); + } +#endif + + //! Copy constructor + limiter_node( const limiter_node& src ) : limiter_node(src.my_graph, src.my_threshold) {} + + //! The interface for accessing internal receiver< DecrementType > that adjusts the count + receiver& decrementer() { return decrement; } + + //! Replace the current successor with this new successor + bool register_successor( successor_type &r ) override { + spin_mutex::scoped_lock lock(my_mutex); + bool was_empty = my_successors.empty(); + my_successors.register_successor(r); + //spawn a forward task if this is the only successor + if ( was_empty && !my_predecessors.empty() && my_count + my_tries < my_threshold ) { + if ( is_graph_active(this->my_graph) ) { + d1::small_object_allocator allocator{}; + typedef forward_task_bypass> task_type; + graph_task* t = allocator.new_object(my_graph, allocator, *this); + spawn_in_graph_arena(graph_reference(), *t); + } + } + return true; + } + + //! Removes a successor from this node + /** r.remove_predecessor(*this) is also called. */ + bool remove_successor( successor_type &r ) override { + // TODO revamp: investigate why qualification is needed for remove_predecessor() call + tbb::detail::d2::remove_predecessor(r, *this); + my_successors.remove_successor(r); + return true; + } + + //! Adds src to the list of cached predecessors. + bool register_predecessor( predecessor_type &src ) override { + spin_mutex::scoped_lock lock(my_mutex); + my_predecessors.add( src ); + if ( my_count + my_tries < my_threshold && !my_successors.empty() && is_graph_active(this->my_graph) ) { + d1::small_object_allocator allocator{}; + typedef forward_task_bypass> task_type; + graph_task* t = allocator.new_object(my_graph, allocator, *this); + spawn_in_graph_arena(graph_reference(), *t); + } + return true; + } + + //! Removes src from the list of cached predecessors. + bool remove_predecessor( predecessor_type &src ) override { + my_predecessors.remove( src ); + return true; + } + +protected: + + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + +private: + //! Puts an item to this receiver + graph_task* try_put_task_impl( const T &t __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo) ) { + { + spin_mutex::scoped_lock lock(my_mutex); + if ( my_count + my_tries >= my_threshold ) + return nullptr; + else + ++my_tries; + } + + graph_task* rtask = my_successors.try_put_task(t __TBB_FLOW_GRAPH_METAINFO_ARG(metainfo)); + if ( !rtask ) { // try_put_task failed. + spin_mutex::scoped_lock lock(my_mutex); + --my_tries; + if (check_conditions() && is_graph_active(this->my_graph)) { + d1::small_object_allocator allocator{}; + typedef forward_task_bypass> task_type; + rtask = allocator.new_object(my_graph, allocator, *this); + } + } + else { + spin_mutex::scoped_lock lock(my_mutex); + ++my_count; + if ( my_future_decrement ) { + if ( my_count > my_future_decrement ) { + my_count -= my_future_decrement; + my_future_decrement = 0; + } + else { + my_future_decrement -= my_count; + my_count = 0; + } + } + --my_tries; + } + return rtask; + } + +protected: + graph_task* try_put_task(const T& t) override { + return try_put_task_impl(t __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const T& t, const message_metainfo& metainfo) override { + return try_put_task_impl(t, metainfo); + } +#endif + + graph& graph_reference() const override { return my_graph; } + + void reset_node( reset_flags f ) override { + my_count = 0; + if ( f & rf_clear_edges ) { + my_predecessors.clear(); + my_successors.clear(); + } + else { + my_predecessors.reset(); + } + decrement.reset_receiver(f); + } +}; // limiter_node + +#include "detail/_flow_graph_join_impl.h" + +template class join_node; + +template +class join_node: public unfolded_join_node { +private: + static const int N = std::tuple_size::value; + using unfolded_type = unfolded_join_node; +public: + typedef OutputTuple output_type; + typedef typename unfolded_type::input_ports_type input_ports_type; + __TBB_NOINLINE_SYM explicit join_node(graph &g) : unfolded_type(g) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_RESERVING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_NOINLINE_SYM join_node(const node_set& nodes, reserving = reserving()) : join_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + __TBB_NOINLINE_SYM join_node(const join_node &other) : unfolded_type(other) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_RESERVING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +}; + +template +class join_node: public unfolded_join_node { +private: + static const int N = std::tuple_size::value; + using unfolded_type = unfolded_join_node; +public: + typedef OutputTuple output_type; + typedef typename unfolded_type::input_ports_type input_ports_type; + __TBB_NOINLINE_SYM explicit join_node(graph &g) : unfolded_type(g) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_QUEUEING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_NOINLINE_SYM join_node(const node_set& nodes, queueing = queueing()) : join_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + __TBB_NOINLINE_SYM join_node(const join_node &other) : unfolded_type(other) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_QUEUEING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +}; + +#if __TBB_CPP20_CONCEPTS_PRESENT +// Helper function which is well-formed only if all of the elements in OutputTuple +// satisfies join_node_function_object +template +void join_node_function_objects_helper( std::index_sequence ) + requires (std::tuple_size_v == sizeof...(Functions)) && + (... && join_node_function_object, K>); + +template +concept join_node_functions = requires { + join_node_function_objects_helper(std::make_index_sequence{}); +}; + +#endif + +// template for key_matching join_node +// tag_matching join_node is a specialization of key_matching, and is source-compatible. +template +class join_node> : public unfolded_join_node> { +private: + static const int N = std::tuple_size::value; + using unfolded_type = unfolded_join_node>; +public: + typedef OutputTuple output_type; + typedef typename unfolded_type::input_ports_type input_ports_type; + +#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING + join_node(graph &g) : unfolded_type(g) {} +#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ + + template > + __TBB_requires(join_node_functions) + __TBB_NOINLINE_SYM join_node(graph& g, Body body, Bodies... bodies) : unfolded_type(g, body, bodies...) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template < +#if (__clang_major__ == 3 && __clang_minor__ == 4) + // clang 3.4 misdeduces 'Args...' for 'node_set' while it can cope with template template parameter. + template class node_set, +#endif + typename... Args, typename... Bodies + > + __TBB_requires((sizeof...(Bodies) == 0) || join_node_functions) + __TBB_NOINLINE_SYM join_node(const node_set& nodes, Bodies... bodies) + : join_node(nodes.graph_reference(), bodies...) { + make_edges_in_order(nodes, *this); + } +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + + __TBB_NOINLINE_SYM join_node(const join_node &other) : unfolded_type(other) { + fgt_multiinput_node( CODEPTR(), FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +}; + +// indexer node +#include "detail/_flow_graph_indexer_impl.h" + +template +__TBB_requires(std::copy_constructible && + (... && std::copy_constructible)) +class indexer_node : public unfolded_indexer_node { +private: + static constexpr std::size_t N = sizeof...(TN) + 1; +public: + using output_type = tagged_msg; + using unfolded_type = unfolded_indexer_node; + + __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + // Copy constructor + __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { + fgt_multiinput_node( CODEPTR(), FLOW_INDEXER_NODE, &this->my_graph, + this->input_ports(), static_cast< sender< output_type > *>(this) ); + } +}; // class indexer_node + +template< typename T > +inline void internal_make_edge( sender &p, receiver &s ) { + register_successor(p, s); + fgt_make_edge( &p, &s ); +} + +//! Makes an edge between a single predecessor and a single successor +template< typename T > +inline void make_edge( sender &p, receiver &s ) { + internal_make_edge( p, s ); +} + +//Makes an edge from port 0 of a multi-output predecessor to port 0 of a multi-input successor. +template< typename T, typename V, + typename = typename T::output_ports_type, typename = typename V::input_ports_type > +inline void make_edge( T& output, V& input) { + make_edge(std::get<0>(output.output_ports()), std::get<0>(input.input_ports())); +} + +//Makes an edge from port 0 of a multi-output predecessor to a receiver. +template< typename T, typename R, + typename = typename T::output_ports_type > +inline void make_edge( T& output, receiver& input) { + make_edge(std::get<0>(output.output_ports()), input); +} + +//Makes an edge from a sender to port 0 of a multi-input successor. +template< typename S, typename V, + typename = typename V::input_ports_type > +inline void make_edge( sender& output, V& input) { + make_edge(output, std::get<0>(input.input_ports())); +} + +template< typename T > +inline void internal_remove_edge( sender &p, receiver &s ) { + remove_successor( p, s ); + fgt_remove_edge( &p, &s ); +} + +//! Removes an edge between a single predecessor and a single successor +template< typename T > +inline void remove_edge( sender &p, receiver &s ) { + internal_remove_edge( p, s ); +} + +//Removes an edge between port 0 of a multi-output predecessor and port 0 of a multi-input successor. +template< typename T, typename V, + typename = typename T::output_ports_type, typename = typename V::input_ports_type > +inline void remove_edge( T& output, V& input) { + remove_edge(std::get<0>(output.output_ports()), std::get<0>(input.input_ports())); +} + +//Removes an edge between port 0 of a multi-output predecessor and a receiver. +template< typename T, typename R, + typename = typename T::output_ports_type > +inline void remove_edge( T& output, receiver& input) { + remove_edge(std::get<0>(output.output_ports()), input); +} +//Removes an edge between a sender and port 0 of a multi-input successor. +template< typename S, typename V, + typename = typename V::input_ports_type > +inline void remove_edge( sender& output, V& input) { + remove_edge(output, std::get<0>(input.input_ports())); +} + +//! Returns a copy of the body from a function or continue node +template< typename Body, typename Node > +Body copy_body( Node &n ) { + return n.template copy_function_object(); +} + +//composite_node +template< typename InputTuple, typename OutputTuple > class composite_node; + +template< typename... InputTypes, typename... OutputTypes> +class composite_node , std::tuple > : public graph_node { + +public: + typedef std::tuple< receiver&... > input_ports_type; + typedef std::tuple< sender&... > output_ports_type; + +private: + std::unique_ptr my_input_ports; + std::unique_ptr my_output_ports; + + static const size_t NUM_INPUTS = sizeof...(InputTypes); + static const size_t NUM_OUTPUTS = sizeof...(OutputTypes); + +protected: + void reset_node(reset_flags) override {} + +public: + composite_node( graph &g ) : graph_node(g) { + fgt_multiinput_multioutput_node( CODEPTR(), FLOW_COMPOSITE_NODE, this, &this->my_graph ); + } + + template + void set_external_ports(T1&& input_ports_tuple, T2&& output_ports_tuple) { + static_assert(NUM_INPUTS == std::tuple_size::value, "number of arguments does not match number of input ports"); + static_assert(NUM_OUTPUTS == std::tuple_size::value, "number of arguments does not match number of output ports"); + + fgt_internal_input_alias_helper::alias_port( this, input_ports_tuple); + fgt_internal_output_alias_helper::alias_port( this, output_ports_tuple); + + my_input_ports.reset( new input_ports_type(std::forward(input_ports_tuple)) ); + my_output_ports.reset( new output_ports_type(std::forward(output_ports_tuple)) ); + } + + template< typename... NodeTypes > + void add_visible_nodes(const NodeTypes&... n) { add_nodes_impl(this, true, n...); } + + template< typename... NodeTypes > + void add_nodes(const NodeTypes&... n) { add_nodes_impl(this, false, n...); } + + + input_ports_type& input_ports() { + __TBB_ASSERT(my_input_ports, "input ports not set, call set_external_ports to set input ports"); + return *my_input_ports; + } + + output_ports_type& output_ports() { + __TBB_ASSERT(my_output_ports, "output ports not set, call set_external_ports to set output ports"); + return *my_output_ports; + } +}; // class composite_node + +//composite_node with only input ports +template< typename... InputTypes> +class composite_node , std::tuple<> > : public graph_node { +public: + typedef std::tuple< receiver&... > input_ports_type; + +private: + std::unique_ptr my_input_ports; + static const size_t NUM_INPUTS = sizeof...(InputTypes); + +protected: + void reset_node(reset_flags) override {} + +public: + composite_node( graph &g ) : graph_node(g) { + fgt_composite( CODEPTR(), this, &g ); + } + + template + void set_external_ports(T&& input_ports_tuple) { + static_assert(NUM_INPUTS == std::tuple_size::value, "number of arguments does not match number of input ports"); + + fgt_internal_input_alias_helper::alias_port( this, input_ports_tuple); + + my_input_ports.reset( new input_ports_type(std::forward(input_ports_tuple)) ); + } + + template< typename... NodeTypes > + void add_visible_nodes(const NodeTypes&... n) { add_nodes_impl(this, true, n...); } + + template< typename... NodeTypes > + void add_nodes( const NodeTypes&... n) { add_nodes_impl(this, false, n...); } + + + input_ports_type& input_ports() { + __TBB_ASSERT(my_input_ports, "input ports not set, call set_external_ports to set input ports"); + return *my_input_ports; + } + +}; // class composite_node + +//composite_nodes with only output_ports +template +class composite_node , std::tuple > : public graph_node { +public: + typedef std::tuple< sender&... > output_ports_type; + +private: + std::unique_ptr my_output_ports; + static const size_t NUM_OUTPUTS = sizeof...(OutputTypes); + +protected: + void reset_node(reset_flags) override {} + +public: + __TBB_NOINLINE_SYM composite_node( graph &g ) : graph_node(g) { + fgt_composite( CODEPTR(), this, &g ); + } + + template + void set_external_ports(T&& output_ports_tuple) { + static_assert(NUM_OUTPUTS == std::tuple_size::value, "number of arguments does not match number of output ports"); + + fgt_internal_output_alias_helper::alias_port( this, output_ports_tuple); + + my_output_ports.reset( new output_ports_type(std::forward(output_ports_tuple)) ); + } + + template + void add_visible_nodes(const NodeTypes&... n) { add_nodes_impl(this, true, n...); } + + template + void add_nodes(const NodeTypes&... n) { add_nodes_impl(this, false, n...); } + + + output_ports_type& output_ports() { + __TBB_ASSERT(my_output_ports, "output ports not set, call set_external_ports to set output ports"); + return *my_output_ports; + } + +}; // class composite_node + +template +class async_body_base: no_assign { +public: + typedef Gateway gateway_type; + + async_body_base(gateway_type *gateway): my_gateway(gateway) { } + void set_gateway(gateway_type *gateway) { + my_gateway = gateway; + } + +protected: + gateway_type *my_gateway; +}; + +template +class async_body: public async_body_base { +private: + Body my_body; + +public: + typedef async_body_base base_type; + typedef Gateway gateway_type; + + async_body(const Body &body, gateway_type *gateway) + : base_type(gateway), my_body(body) { } + + void operator()( const Input &v, Ports & ) noexcept(noexcept(tbb::detail::invoke(my_body, v, std::declval()))) { + tbb::detail::invoke(my_body, v, *this->my_gateway); + } + + Body get_body() { return my_body; } +}; + +//! Implements async node +template < typename Input, typename Output, + typename Policy = queueing_lightweight > + __TBB_requires(std::default_initializable && std::copy_constructible) +class async_node + : public multifunction_node< Input, std::tuple< Output >, Policy >, public sender< Output > +{ + typedef multifunction_node< Input, std::tuple< Output >, Policy > base_type; + typedef multifunction_input< + Input, typename base_type::output_ports_type, Policy, cache_aligned_allocator> mfn_input_type; + +public: + typedef Input input_type; + typedef Output output_type; + typedef receiver receiver_type; + typedef receiver successor_type; + typedef sender predecessor_type; + typedef receiver_gateway gateway_type; + typedef async_body_base async_body_base_type; + typedef typename base_type::output_ports_type output_ports_type; + +private: + class receiver_gateway_impl: public receiver_gateway { + public: + receiver_gateway_impl(async_node* node): my_node(node) {} + void reserve_wait() override { + fgt_async_reserve(static_cast(my_node), &my_node->my_graph); + my_node->my_graph.reserve_wait(); + } + + void release_wait() override { + async_node* n = my_node; + graph* g = &n->my_graph; + g->release_wait(); + fgt_async_commit(static_cast(n), g); + } + + //! Implements gateway_type::try_put for an external activity to submit a message to FG + bool try_put(const Output &i) override { + return my_node->try_put_impl(i); + } + + private: + async_node* my_node; + } my_gateway; + + //The substitute of 'this' for member construction, to prevent compiler warnings + async_node* self() { return this; } + + //! Implements gateway_type::try_put for an external activity to submit a message to FG + bool try_put_impl(const Output &i) { + multifunction_output &port_0 = output_port<0>(*this); + broadcast_cache& port_successors = port_0.successors(); + fgt_async_try_put_begin(this, &port_0); + // TODO revamp: change to std::list + graph_task_list tasks; + bool is_at_least_one_put_successful = port_successors.gather_successful_try_puts(i, tasks); + __TBB_ASSERT( is_at_least_one_put_successful || tasks.empty(), + "Return status is inconsistent with the method operation." ); + + while( !tasks.empty() ) { + enqueue_in_graph_arena(this->my_graph, tasks.pop_front()); + } + fgt_async_try_put_end(this, &port_0); + return is_at_least_one_put_successful; + } + +public: + template + __TBB_requires(async_node_body) + __TBB_NOINLINE_SYM async_node( + graph &g, size_t concurrency, + Body body, Policy = Policy(), node_priority_t a_priority = no_priority + ) : base_type( + g, concurrency, + async_body + (body, &my_gateway), a_priority ), my_gateway(self()) { + fgt_multioutput_node_with_body<1>( + CODEPTR(), FLOW_ASYNC_NODE, + &this->my_graph, static_cast *>(this), + this->output_ports(), this->my_body + ); + } + + template + __TBB_requires(async_node_body) + __TBB_NOINLINE_SYM async_node(graph& g, size_t concurrency, Body body, node_priority_t a_priority) + : async_node(g, concurrency, body, Policy(), a_priority) {} + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + __TBB_requires(async_node_body) + __TBB_NOINLINE_SYM async_node( + const node_set& nodes, size_t concurrency, Body body, + Policy = Policy(), node_priority_t a_priority = no_priority ) + : async_node(nodes.graph_reference(), concurrency, body, a_priority) { + make_edges_in_order(nodes, *this); + } + + template + __TBB_requires(async_node_body) + __TBB_NOINLINE_SYM async_node(const node_set& nodes, size_t concurrency, Body body, node_priority_t a_priority) + : async_node(nodes, concurrency, body, Policy(), a_priority) {} +#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + + __TBB_NOINLINE_SYM async_node( const async_node &other ) : base_type(other), sender(), my_gateway(self()) { + static_cast(this->my_body->get_body_ptr())->set_gateway(&my_gateway); + static_cast(this->my_init_body->get_body_ptr())->set_gateway(&my_gateway); + + fgt_multioutput_node_with_body<1>( CODEPTR(), FLOW_ASYNC_NODE, + &this->my_graph, static_cast *>(this), + this->output_ports(), this->my_body ); + } + + gateway_type& gateway() { + return my_gateway; + } + + // Define sender< Output > + + //! Add a new successor to this node + bool register_successor(successor_type&) override { + __TBB_ASSERT(false, "Successors must be registered only via ports"); + return false; + } + + //! Removes a successor from this node + bool remove_successor(successor_type&) override { + __TBB_ASSERT(false, "Successors must be removed only via ports"); + return false; + } + + template + Body copy_function_object() { + typedef multifunction_body mfn_body_type; + typedef async_body async_body_type; + mfn_body_type &body_ref = *this->my_body; + async_body_type ab = *static_cast(dynamic_cast< multifunction_body_leaf & >(body_ref).get_body_ptr()); + return ab.get_body(); + } + +protected: + + void reset_node( reset_flags f) override { + base_type::reset_node(f); + } +}; + +#include "detail/_flow_graph_node_set_impl.h" + +template< typename T > +class overwrite_node : public graph_node, public receiver, public sender { +public: + typedef T input_type; + typedef T output_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; + + __TBB_NOINLINE_SYM explicit overwrite_node(graph &g) + : graph_node(g), my_successors(this), my_buffer_is_valid(false) + { + fgt_node( CODEPTR(), FLOW_OVERWRITE_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + overwrite_node(const node_set& nodes) : overwrite_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + //! Copy constructor; doesn't take anything from src; default won't work + __TBB_NOINLINE_SYM overwrite_node( const overwrite_node& src ) : overwrite_node(src.my_graph) {} + + ~overwrite_node() {} + + bool register_successor( successor_type &s ) override { + spin_mutex::scoped_lock l( my_mutex ); + if (my_buffer_is_valid && is_graph_active( my_graph )) { + // We have a valid value that must be forwarded immediately. + if (s.is_continue_receiver()) { + // try_put can never fail, since continue_receivers always accept + my_successors.register_successor( s ); + s.try_put( my_buffer ); + return true; + } + bool ret = s.try_put( my_buffer ); + if ( ret ) { + // We add the successor that accepted our put + my_successors.register_successor( s ); + } else { + // In case of reservation a race between the moment of reservation and register_successor can appear, + // because failed reserve does not mean that register_successor is not ready to put a message immediately. + // We have some sort of infinite loop: reserving node tries to set pull state for the edge, + // but overwrite_node tries to return push state back. That is why we have to break this loop with task creation. + d1::small_object_allocator allocator{}; + typedef register_predecessor_task task_type; + graph_task* t = allocator.new_object(graph_reference(), allocator, *this, s); + spawn_in_graph_arena( my_graph, *t ); + } + } else { + // No valid value yet, just add as successor + my_successors.register_successor( s ); + } + return true; + } + + bool remove_successor( successor_type &s ) override { + spin_mutex::scoped_lock l( my_mutex ); + my_successors.remove_successor(s); + return true; + } + + bool try_get( input_type &v ) override { + spin_mutex::scoped_lock l( my_mutex ); + if ( my_buffer_is_valid ) { + v = my_buffer; + return true; + } + return false; + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + bool try_get( input_type &v, message_metainfo& metainfo ) override { + spin_mutex::scoped_lock l( my_mutex ); + if (my_buffer_is_valid) { + v = my_buffer; + metainfo = my_buffered_metainfo; + + // Since the successor of the node will use move semantics while wrapping the metainfo + // that is designed to transfer the ownership of the value from single-push buffer to the task + // It is required to reserve one more reference here because the value keeps in the buffer + // and the ownership is not transferred + for (auto msg_waiter : metainfo.waiters()) { + msg_waiter->reserve(1); + } + return true; + } + return false; + } +#endif + + //! Reserves an item + bool try_reserve( T &v ) override { + return try_get(v); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT +private: + bool try_reserve(T& v, message_metainfo& metainfo) override { + spin_mutex::scoped_lock l( my_mutex ); + if (my_buffer_is_valid) { + v = my_buffer; + metainfo = my_buffered_metainfo; + return true; + } + return false; + } +public: +#endif + + //! Releases the reserved item + bool try_release() override { return true; } + + //! Consumes the reserved item + bool try_consume() override { return true; } + + bool is_valid() { + spin_mutex::scoped_lock l( my_mutex ); + return my_buffer_is_valid; + } + + void clear() { + spin_mutex::scoped_lock l( my_mutex ); + my_buffer_is_valid = false; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + for (auto msg_waiter : my_buffered_metainfo.waiters()) { + msg_waiter->release(1); + } + my_buffered_metainfo = message_metainfo{}; +#endif + } + +protected: + + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + graph_task* try_put_task( const input_type &v ) override { + spin_mutex::scoped_lock l( my_mutex ); + return try_put_task_impl(v __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const input_type& v, const message_metainfo& metainfo) override { + spin_mutex::scoped_lock l( my_mutex ); + return try_put_task_impl(v, metainfo); + } +#endif + + graph_task * try_put_task_impl(const input_type &v __TBB_FLOW_GRAPH_METAINFO_ARG(const message_metainfo& metainfo)) { + my_buffer = v; + my_buffer_is_valid = true; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + // Since the new item is pushed to the buffer - reserving the waiters + for (auto msg_waiter : metainfo.waiters()) { + msg_waiter->reserve(1); + } + + // Since the item is taken out from the buffer - releasing the stored waiters + for (auto msg_waiter : my_buffered_metainfo.waiters()) { + msg_waiter->release(1); + } + + my_buffered_metainfo = metainfo; +#endif + graph_task* rtask = my_successors.try_put_task(v __TBB_FLOW_GRAPH_METAINFO_ARG(my_buffered_metainfo) ); + if (!rtask) rtask = SUCCESSFULLY_ENQUEUED; + return rtask; + } + + graph& graph_reference() const override { + return my_graph; + } + + //! Breaks an infinite loop between the node reservation and register_successor call + struct register_predecessor_task : public graph_task { + register_predecessor_task( + graph& g, d1::small_object_allocator& allocator, predecessor_type& owner, successor_type& succ) + : graph_task(g, allocator), o(owner), s(succ) {}; + + d1::task* execute(d1::execution_data& ed) override { + // TODO revamp: investigate why qualification is needed for register_successor() call + using tbb::detail::d2::register_predecessor; + using tbb::detail::d2::register_successor; + if ( !register_predecessor(s, o) ) { + register_successor(o, s); + } + finalize(ed); + return nullptr; + } + + d1::task* cancel(d1::execution_data& ed) override { + finalize(ed); + return nullptr; + } + + predecessor_type& o; + successor_type& s; + }; + + spin_mutex my_mutex; + broadcast_cache< input_type, null_rw_mutex > my_successors; + input_type my_buffer; +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + message_metainfo my_buffered_metainfo; +#endif + bool my_buffer_is_valid; + + void reset_node( reset_flags f) override { + my_buffer_is_valid = false; + if (f&rf_clear_edges) { + my_successors.clear(); + } + } +}; // overwrite_node + +template< typename T > +class write_once_node : public overwrite_node { +public: + typedef T input_type; + typedef T output_type; + typedef overwrite_node base_type; + typedef typename receiver::predecessor_type predecessor_type; + typedef typename sender::successor_type successor_type; + + //! Constructor + __TBB_NOINLINE_SYM explicit write_once_node(graph& g) : base_type(g) { + fgt_node( CODEPTR(), FLOW_WRITE_ONCE_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + template + write_once_node(const node_set& nodes) : write_once_node(nodes.graph_reference()) { + make_edges_in_order(nodes, *this); + } +#endif + + //! Copy constructor: call base class copy constructor + __TBB_NOINLINE_SYM write_once_node( const write_once_node& src ) : base_type(src) { + fgt_node( CODEPTR(), FLOW_WRITE_ONCE_NODE, &(this->my_graph), + static_cast *>(this), + static_cast *>(this) ); + } + +protected: + template< typename R, typename B > friend class run_and_put_task; + template friend class broadcast_cache; + template friend class round_robin_cache; + graph_task *try_put_task( const T &v ) override { + spin_mutex::scoped_lock l( this->my_mutex ); + return this->my_buffer_is_valid ? nullptr : this->try_put_task_impl(v __TBB_FLOW_GRAPH_METAINFO_ARG(message_metainfo{})); + } + +#if __TBB_PREVIEW_FLOW_GRAPH_TRY_PUT_AND_WAIT + graph_task* try_put_task(const T& v, const message_metainfo& metainfo) override { + spin_mutex::scoped_lock l( this->my_mutex ); + return this->my_buffer_is_valid ? nullptr : this->try_put_task_impl(v, metainfo); + } +#endif +}; // write_once_node + +inline void set_name(const graph& g, const char *name) { + fgt_graph_desc(&g, name); +} + +template +inline void set_name(const input_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const function_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const continue_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const broadcast_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const buffer_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const queue_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const sequencer_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const priority_queue_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const limiter_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const join_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const indexer_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const overwrite_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const write_once_node& node, const char *name) { + fgt_node_desc(&node, name); +} + +template +inline void set_name(const multifunction_node& node, const char *name) { + fgt_multioutput_node_desc(&node, name); +} + +template +inline void set_name(const split_node& node, const char *name) { + fgt_multioutput_node_desc(&node, name); +} + +template< typename InputTuple, typename OutputTuple > +inline void set_name(const composite_node& node, const char *name) { + fgt_multiinput_multioutput_node_desc(&node, name); +} + +template +inline void set_name(const async_node& node, const char *name) +{ + fgt_multioutput_node_desc(&node, name); +} + +} // d2 +} // detail +} // tbb + +#if __TBB_PREVIEW_FLOW_GRAPH_RESOURCE_LIMITING +#include "detail/_flow_graph_resource_limiting.h" +#endif + +// Include deduction guides for node classes +#include "detail/_flow_graph_nodes_deduction.h" + +namespace tbb { +namespace flow { +inline namespace v1 { + using detail::d2::receiver; + using detail::d2::sender; + + using detail::d2::serial; + using detail::d2::unlimited; + + using detail::d2::reset_flags; + using detail::d2::rf_reset_protocol; + using detail::d2::rf_reset_bodies; + using detail::d2::rf_clear_edges; + + using detail::d2::graph; + using detail::d2::graph_node; + using detail::d2::continue_msg; + + using detail::d2::input_node; + using detail::d2::function_node; + using detail::d2::multifunction_node; + using detail::d2::split_node; + using detail::d2::output_port; + using detail::d2::indexer_node; + using detail::d2::tagged_msg; + using detail::d2::cast_to; + using detail::d2::is_a; + using detail::d2::continue_node; + using detail::d2::overwrite_node; + using detail::d2::write_once_node; + using detail::d2::broadcast_node; + using detail::d2::buffer_node; + using detail::d2::queue_node; + using detail::d2::sequencer_node; + using detail::d2::priority_queue_node; + using detail::d2::limiter_node; + using namespace detail::d2::graph_policy_namespace; + using detail::d2::join_node; + using detail::d2::input_port; + using detail::d2::copy_body; + using detail::d2::make_edge; + using detail::d2::remove_edge; + using detail::d2::tag_value; + using detail::d2::composite_node; + using detail::d2::async_node; + using detail::d2::node_priority_t; + using detail::d2::no_priority; + +#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET + using detail::d2::follows; + using detail::d2::precedes; + using detail::d2::make_node_set; + using detail::d2::make_edges; +#endif + +#if __TBB_PREVIEW_FLOW_GRAPH_RESOURCE_LIMITING + using detail::d2::resource_limiter; + using detail::d2::resource_limited_node; +#endif +} // v1 +} // flow + + using detail::d1::flow_control; + +namespace profiling { + using detail::d2::set_name; +} // profiling + +} // tbb + + +#if TBB_USE_PROFILING_TOOLS && ( __unix__ || __APPLE__ ) + // We don't do pragma pop here, since it still gives warning on the USER side + #undef __TBB_NOINLINE_SYM +#endif + +#endif // __TBB_flow_graph_H diff --git a/src/3rd party/oneapi/tbb/flow_graph_abstractions.h b/src/3rd party/oneapi/tbb/flow_graph_abstractions.h new file mode 100644 index 0000000000..329e75c43e --- /dev/null +++ b/src/3rd party/oneapi/tbb/flow_graph_abstractions.h @@ -0,0 +1,51 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_flow_graph_abstractions_H +#define __TBB_flow_graph_abstractions_H + +namespace tbb { +namespace detail { +namespace d2 { + +//! Pure virtual template classes that define interfaces for async communication +class graph_proxy { +public: + //! Inform a graph that messages may come from outside, to prevent premature graph completion + virtual void reserve_wait() = 0; + + //! Inform a graph that a previous call to reserve_wait is no longer in effect + virtual void release_wait() = 0; + + virtual ~graph_proxy() {} +}; + +template +class receiver_gateway : public graph_proxy { +public: + //! Type of inputing data into FG. + typedef Input input_type; + + //! Submit signal from an asynchronous activity to FG. + virtual bool try_put(const input_type&) = 0; +}; + +} // d2 + + +} // detail +} // tbb +#endif diff --git a/src/3rd party/oneapi/tbb/global_control.h b/src/3rd party/oneapi/tbb/global_control.h new file mode 100644 index 0000000000..cfff3d2272 --- /dev/null +++ b/src/3rd party/oneapi/tbb/global_control.h @@ -0,0 +1,229 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + Copyright (c) 2025 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_global_control_H +#define __TBB_global_control_H + +#include "detail/_config.h" + +#include "detail/_assert.h" +#include "detail/_attach.h" +#include "detail/_exception.h" +#include "detail/_namespace_injection.h" +#include "detail/_template_helpers.h" + +#include +#include // std::nothrow_t + +namespace tbb { +namespace detail { + +namespace d1 { +class global_control; +class task_scheduler_handle; +} + +namespace r1 { +TBB_EXPORT void __TBB_EXPORTED_FUNC create(d1::global_control&); +TBB_EXPORT void __TBB_EXPORTED_FUNC destroy(d1::global_control&); +TBB_EXPORT std::size_t __TBB_EXPORTED_FUNC global_control_active_value(int); +struct global_control_impl; +struct control_storage_comparator; +void release_impl(d1::task_scheduler_handle& handle); +bool finalize_impl(d1::task_scheduler_handle& handle); +TBB_EXPORT void __TBB_EXPORTED_FUNC get(d1::task_scheduler_handle&); +TBB_EXPORT bool __TBB_EXPORTED_FUNC finalize(d1::task_scheduler_handle&, std::intptr_t mode); + +//! Set assertion handler and return its previous value. +//! The handler should not return. If it eventually returns, the behavior is runtime-undefined. +TBB_EXPORT assertion_handler_type __TBB_EXPORTED_FUNC +set_assertion_handler(assertion_handler_type new_handler) noexcept; + +//! Return the current assertion handler. +TBB_EXPORT assertion_handler_type __TBB_EXPORTED_FUNC get_assertion_handler() noexcept; +} + +namespace d1 { + +class global_control { +public: + enum parameter { + max_allowed_parallelism, + thread_stack_size, + terminate_on_exception, + scheduler_handle, // not a public parameter +#if __TBB_PREVIEW_PARALLEL_PHASE + leave_policy, +#endif + parameter_max // insert new parameters above this point + }; + + global_control(parameter p, std::size_t value) : + my_value(value), my_reserved(), my_param(p) { + suppress_unused_warning(my_reserved); + __TBB_ASSERT(my_param < parameter_max, "Invalid parameter"); +#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) + // For Windows 8 Store* apps it's impossible to set stack size + if (p==thread_stack_size) + return; +#elif __TBB_x86_64 && (_WIN32 || _WIN64) + if (p==thread_stack_size) + __TBB_ASSERT_RELEASE((unsigned)value == value, "Stack size is limited to unsigned int range"); +#endif + if (my_param==max_allowed_parallelism) + __TBB_ASSERT_RELEASE(my_value>0, "max_allowed_parallelism cannot be 0."); + r1::create(*this); + } + +#if __TBB_PREVIEW_PARALLEL_PHASE + //! Overload the constructor for enum types to avoid forcing users to cast them to size_t + template::value>::type> + global_control(parameter p, T value) + : global_control(p, static_cast(value)) + {} +#endif + + ~global_control() { + __TBB_ASSERT(my_param < parameter_max, "Invalid parameter"); +#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) + // For Windows 8 Store* apps it's impossible to set stack size + if (my_param==thread_stack_size) + return; +#endif + r1::destroy(*this); + } + + static std::size_t active_value(parameter p) { + return r1::global_control_active_value((int)p); + } + +private: + std::size_t my_value; + std::intptr_t my_reserved; // TODO: substitution of global_control* not to break backward compatibility + parameter my_param; + + friend struct r1::global_control_impl; + friend struct r1::control_storage_comparator; +}; + +//! Finalization options. +//! Outside of the class to avoid extensive friendship. +__TBB_GLOBAL_VAR constexpr std::intptr_t release_nothrowing = 0; +__TBB_GLOBAL_VAR constexpr std::intptr_t finalize_nothrowing = 1; +__TBB_GLOBAL_VAR constexpr std::intptr_t finalize_throwing = 2; + +//! User side wrapper for a task scheduler lifetime control object +class task_scheduler_handle { +public: + //! Creates an empty task_scheduler_handle + task_scheduler_handle() = default; + + //! Creates an attached instance of task_scheduler_handle + task_scheduler_handle(attach) { + r1::get(*this); + } + + //! Release a reference if any + ~task_scheduler_handle() { + release(); + } + + //! No copy + task_scheduler_handle(const task_scheduler_handle& other) = delete; + task_scheduler_handle& operator=(const task_scheduler_handle& other) = delete; + + //! Move only + task_scheduler_handle(task_scheduler_handle&& other) noexcept { + std::swap(m_ctl, other.m_ctl); + } + task_scheduler_handle& operator=(task_scheduler_handle&& other) noexcept { + std::swap(m_ctl, other.m_ctl); + return *this; + }; + + //! Checks if the task_scheduler_handle is empty + explicit operator bool() const noexcept { + return m_ctl != nullptr; + } + + //! Release the reference and deactivate handle + void release() { + if (m_ctl != nullptr) { + r1::finalize(*this, release_nothrowing); + m_ctl = nullptr; + } + } + +private: + friend void r1::release_impl(task_scheduler_handle& handle); + friend bool r1::finalize_impl(task_scheduler_handle& handle); + friend void __TBB_EXPORTED_FUNC r1::get(task_scheduler_handle&); + + friend void finalize(task_scheduler_handle&); + friend bool finalize(task_scheduler_handle&, const std::nothrow_t&) noexcept; + + global_control* m_ctl{nullptr}; +}; + +#if TBB_USE_EXCEPTIONS +//! Waits for worker threads termination. Throws exception on error. +inline void finalize(task_scheduler_handle& handle) { + try_call([&] { + if (handle.m_ctl != nullptr) { + bool finalized = r1::finalize(handle, finalize_throwing); + __TBB_ASSERT_EX(finalized, "r1::finalize did not respect finalize_throwing ?"); + + } + }).on_completion([&] { + __TBB_ASSERT(!handle, "The handle should be empty after finalize"); + }); +} +#endif +//! Waits for worker threads termination. Returns false on error. +inline bool finalize(task_scheduler_handle& handle, const std::nothrow_t&) noexcept { + bool finalized = true; + if (handle.m_ctl != nullptr) { + finalized = r1::finalize(handle, finalize_nothrowing); + } + __TBB_ASSERT(!handle, "The handle should be empty after finalize"); + return finalized; +} + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::global_control; +using detail::d1::attach; +using detail::d1::finalize; +using detail::d1::task_scheduler_handle; +using detail::r1::unsafe_wait; +} // namespace v1 + +namespace ext { +inline namespace v1 { +#if !__TBB_DISABLE_SPEC_EXTENSIONS +using ::tbb::detail::r1::assertion_handler_type; +using ::tbb::detail::r1::set_assertion_handler; +using ::tbb::detail::r1::get_assertion_handler; +#endif +} // inline namespace v1 +} // namespace ext + +} // namespace tbb + +#endif // __TBB_global_control_H diff --git a/src/3rd party/oneapi/tbb/info.h b/src/3rd party/oneapi/tbb/info.h new file mode 100644 index 0000000000..38b21f08fd --- /dev/null +++ b/src/3rd party/oneapi/tbb/info.h @@ -0,0 +1,179 @@ +/* + Copyright (c) 2019-2022 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_info_H +#define __TBB_info_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_utils.h" +#include "version.h" + +#include +#include +#include + +namespace tbb { +namespace detail { + +namespace d1{ + +using numa_node_id = int; +using core_type_id = int; + +// TODO: consider version approach to resolve backward compatibility potential issues. +struct constraints { +#if !__TBB_CPP20_PRESENT + constraints(numa_node_id id = -1, int maximal_concurrency = -1) + : numa_id(id) + , max_concurrency(maximal_concurrency) + {} +#endif /*!__TBB_CPP20_PRESENT*/ + + constraints& set_numa_id(numa_node_id id) { + numa_id = id; + return *this; + } + constraints& set_max_concurrency(int maximal_concurrency) { + max_concurrency = maximal_concurrency; + return *this; + } + constraints& set_core_type(core_type_id id) { + core_type = id; + return *this; + } + constraints& set_max_threads_per_core(int threads_number) { + max_threads_per_core = threads_number; + return *this; + } + + numa_node_id numa_id = -1; + int max_concurrency = -1; + core_type_id core_type = -1; + int max_threads_per_core = -1; +}; + +} // namespace d1 + +namespace r1 { +TBB_EXPORT unsigned __TBB_EXPORTED_FUNC numa_node_count(); +TBB_EXPORT void __TBB_EXPORTED_FUNC fill_numa_indices(int* index_array); +TBB_EXPORT int __TBB_EXPORTED_FUNC numa_default_concurrency(int numa_id); + +// Reserved fields are required to save binary backward compatibility in case of future changes. +// They must be defined to 0 at this moment. +TBB_EXPORT unsigned __TBB_EXPORTED_FUNC core_type_count(intptr_t reserved = 0); +TBB_EXPORT void __TBB_EXPORTED_FUNC fill_core_type_indices(int* index_array, intptr_t reserved = 0); + +TBB_EXPORT int __TBB_EXPORTED_FUNC constraints_default_concurrency(const d1::constraints& c, intptr_t reserved = 0); +TBB_EXPORT int __TBB_EXPORTED_FUNC constraints_threads_per_core(const d1::constraints& c, intptr_t reserved = 0); +} // namespace r1 + +namespace d1 { + +inline std::vector numa_nodes() { + std::vector node_indices(r1::numa_node_count()); + r1::fill_numa_indices(node_indices.data()); + return node_indices; +} + +inline int default_concurrency(numa_node_id id = -1) { + return r1::numa_default_concurrency(id); +} + +inline std::vector core_types() { + std::vector core_type_indexes(r1::core_type_count()); + r1::fill_core_type_indices(core_type_indexes.data()); + return core_type_indexes; +} + +inline int default_concurrency(constraints c) { + if (c.max_concurrency > 0) { return c.max_concurrency; } + return r1::constraints_default_concurrency(c); +} + +#if __TBB_PREVIEW_TASK_ARENA_CORE_TYPE_SELECTOR +// Call a custom selector on the available core type(s) and encode those selected +template +inline core_type_id apply_core_type_selector(Selector selector) { + constexpr core_type_id automatic = -1; + + auto ids = core_types(); + size_t total = ids.size(); + if (total < 2) { + // Not enough core types to select from, so use the default + return automatic; + } + + int max_score = 0, max_score_id = -1, num_zero_scores = 0; + std::vector selected_core_types; + for (size_t index = 0; index < total; ++index) { + int score = selector(std::make_tuple(ids[index], index, total)); + if (score > 0) { + selected_core_types.push_back(ids[index]); + } + else if (score == 0) { + ++num_zero_scores; + } + + if (TBB_runtime_interface_version() < 12180) { + if (score > max_score) { + max_score = score; + max_score_id = ids[index]; + } + } + } + if (TBB_runtime_interface_version() < 12180) { + // No runtime multi core type support, so select all or one + if (selected_core_types.size() + num_zero_scores == total) { + selected_core_types.clear(); // all + } + else if (!selected_core_types.empty()) { + selected_core_types = { max_score_id }; // the one with the highest score + } + } + return multi_core_type_codec::encode(selected_core_types); +} + +template +inline int default_concurrency(constraints c, Selector selector) { + constexpr core_type_id selectable = -2; + if (c.core_type == selectable) { + c.core_type = apply_core_type_selector(selector); + } + return default_concurrency(c); +} +#endif + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::numa_node_id; +using detail::d1::core_type_id; + +namespace info { +using detail::d1::numa_nodes; +using detail::d1::core_types; + +using detail::d1::default_concurrency; +} // namespace info +} // namespace v1 + +} // namespace tbb + +#endif /*__TBB_info_H*/ diff --git a/src/3rd party/oneapi/tbb/memory_pool.h b/src/3rd party/oneapi/tbb/memory_pool.h new file mode 100644 index 0000000000..5ece879002 --- /dev/null +++ b/src/3rd party/oneapi/tbb/memory_pool.h @@ -0,0 +1,272 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_memory_pool_H +#define __TBB_memory_pool_H + +#if !TBB_PREVIEW_MEMORY_POOL +#error Set TBB_PREVIEW_MEMORY_POOL to include memory_pool.h +#endif +/** @file */ + +#include "scalable_allocator.h" + +#include // std::bad_alloc +#include // std::runtime_error, std::invalid_argument +#include // std::forward + + +#if __TBB_EXTRA_DEBUG +#define __TBBMALLOC_ASSERT ASSERT +#else +#define __TBBMALLOC_ASSERT(a,b) ((void)0) +#endif + +namespace tbb { +namespace detail { +namespace d1 { + +//! Base of thread-safe pool allocator for variable-size requests +class pool_base : no_copy { + // Pool interface is separate from standard allocator classes because it has + // to maintain internal state, no copy or assignment. Move and swap are possible. +public: + //! Reset pool to reuse its memory (free all objects at once) + void recycle() { rml::pool_reset(my_pool); } + + //! The "malloc" analogue to allocate block of memory of size bytes + void *malloc(size_t size) { return rml::pool_malloc(my_pool, size); } + + //! The "free" analogue to discard a previously allocated piece of memory. + void free(void* ptr) { rml::pool_free(my_pool, ptr); } + + //! The "realloc" analogue complementing pool_malloc. + // Enables some low-level optimization possibilities + void *realloc(void* ptr, size_t size) { + return rml::pool_realloc(my_pool, ptr, size); + } + +protected: + //! destroy pool - must be called in a child class + void destroy() { rml::pool_destroy(my_pool); } + + rml::MemoryPool *my_pool; +}; + +#if _MSC_VER && !defined(__INTEL_COMPILER) + // Workaround for erroneous "unreferenced parameter" warning in method destroy. + #pragma warning (push) + #pragma warning (disable: 4100) +#endif + +//! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 +/** @ingroup memory_allocation */ +template +class memory_pool_allocator { +protected: + typedef P pool_type; + pool_type *my_pool; + template + friend class memory_pool_allocator; + template + friend bool operator==( const memory_pool_allocator& a, const memory_pool_allocator& b); + template + friend bool operator!=( const memory_pool_allocator& a, const memory_pool_allocator& b); +public: + typedef T value_type; + typedef value_type* pointer; + typedef const value_type* const_pointer; + typedef value_type& reference; + typedef const value_type& const_reference; + typedef size_t size_type; + typedef ptrdiff_t difference_type; + template struct rebind { + typedef memory_pool_allocator other; + }; + + explicit memory_pool_allocator(pool_type &pool) noexcept : my_pool(&pool) {} + memory_pool_allocator(const memory_pool_allocator& src) noexcept : my_pool(src.my_pool) {} + template + memory_pool_allocator(const memory_pool_allocator& src) noexcept : my_pool(src.my_pool) {} + + pointer address(reference x) const { return &x; } + const_pointer address(const_reference x) const { return &x; } + + //! Allocate space for n objects. + pointer allocate( size_type n, const void* /*hint*/ = nullptr) { + pointer p = static_cast( my_pool->malloc( n*sizeof(value_type) ) ); + if (!p) + throw_exception(std::bad_alloc()); + return p; + } + //! Free previously allocated block of memory. + void deallocate( pointer p, size_type ) { + my_pool->free(p); + } + //! Largest value for which method allocate might succeed. + size_type max_size() const noexcept { + size_type max = static_cast(-1) / sizeof (value_type); + return (max > 0 ? max : 1); + } + //! Copy-construct value at location pointed to by p. + + template + void construct(U *p, Args&&... args) + { ::new((void *)p) U(std::forward(args)...); } + + //! Destroy value at location pointed to by p. + void destroy( pointer p ) { p->~value_type(); } + +}; + +#if _MSC_VER && !defined(__INTEL_COMPILER) + #pragma warning (pop) +#endif // warning 4100 is back + +//! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1 +/** @ingroup memory_allocation */ +template +class memory_pool_allocator { +public: + typedef P pool_type; + typedef void* pointer; + typedef const void* const_pointer; + typedef void value_type; + template struct rebind { + typedef memory_pool_allocator other; + }; + + explicit memory_pool_allocator( pool_type &pool) noexcept : my_pool(&pool) {} + memory_pool_allocator( const memory_pool_allocator& src) noexcept : my_pool(src.my_pool) {} + template + memory_pool_allocator(const memory_pool_allocator& src) noexcept : my_pool(src.my_pool) {} + +protected: + pool_type *my_pool; + template + friend class memory_pool_allocator; + template + friend bool operator==( const memory_pool_allocator& a, const memory_pool_allocator& b); + template + friend bool operator!=( const memory_pool_allocator& a, const memory_pool_allocator& b); +}; + +template +inline bool operator==( const memory_pool_allocator& a, const memory_pool_allocator& b) {return a.my_pool==b.my_pool;} + +template +inline bool operator!=( const memory_pool_allocator& a, const memory_pool_allocator& b) {return a.my_pool!=b.my_pool;} + +//! Thread-safe growable pool allocator for variable-size requests +template +class memory_pool : public pool_base { + Alloc my_alloc; // TODO: base-class optimization + static void *allocate_request(intptr_t pool_id, size_t & bytes); + static int deallocate_request(intptr_t pool_id, void*, size_t raw_bytes); + +public: + //! construct pool with underlying allocator + explicit memory_pool(const Alloc &src = Alloc()); + + //! destroy pool + ~memory_pool() { destroy(); } // call the callbacks first and destroy my_alloc latter +}; + +class fixed_pool : public pool_base { + void *my_buffer; + size_t my_size; + inline static void *allocate_request(intptr_t pool_id, size_t & bytes); + +public: + //! construct pool with underlying allocator + inline fixed_pool(void *buf, size_t size); + //! destroy pool + ~fixed_pool() { destroy(); } +}; + +//////////////// Implementation /////////////// + +template +memory_pool::memory_pool(const Alloc &src) : my_alloc(src) { + rml::MemPoolPolicy args(allocate_request, deallocate_request, + sizeof(typename Alloc::value_type)); + rml::MemPoolError res = rml::pool_create_v1(intptr_t(this), &args, &my_pool); + if (res!=rml::POOL_OK) + throw_exception(std::runtime_error("Can't create pool")); +} +template +void *memory_pool::allocate_request(intptr_t pool_id, size_t & bytes) { + memory_pool &self = *reinterpret_cast*>(pool_id); + const size_t unit_size = sizeof(typename Alloc::value_type); + __TBBMALLOC_ASSERT( 0 == bytes%unit_size, nullptr); + void *ptr; +#if TBB_USE_EXCEPTIONS + try { +#endif + ptr = self.my_alloc.allocate( bytes/unit_size ); +#if TBB_USE_EXCEPTIONS + } catch(...) { + return nullptr; + } +#endif + return ptr; +} +#if __TBB_MSVC_UNREACHABLE_CODE_IGNORED + // Workaround for erroneous "unreachable code" warning in the template below. + // Specific for VC++ 17-18 compiler + #pragma warning (push) + #pragma warning (disable: 4702) +#endif +template +int memory_pool::deallocate_request(intptr_t pool_id, void* raw_ptr, size_t raw_bytes) { + memory_pool &self = *reinterpret_cast*>(pool_id); + const size_t unit_size = sizeof(typename Alloc::value_type); + __TBBMALLOC_ASSERT( 0 == raw_bytes%unit_size, nullptr); + self.my_alloc.deallocate( static_cast(raw_ptr), raw_bytes/unit_size ); + return 0; +} +#if __TBB_MSVC_UNREACHABLE_CODE_IGNORED + #pragma warning (pop) +#endif +inline fixed_pool::fixed_pool(void *buf, size_t size) : my_buffer(buf), my_size(size) { + if (!buf || !size) + // TODO: improve support for mode with exceptions disabled + throw_exception(std::invalid_argument("Zero in parameter is invalid")); + rml::MemPoolPolicy args(allocate_request, nullptr, size, /*fixedPool=*/true); + rml::MemPoolError res = rml::pool_create_v1(intptr_t(this), &args, &my_pool); + if (res!=rml::POOL_OK) + throw_exception(std::runtime_error("Can't create pool")); +} +inline void *fixed_pool::allocate_request(intptr_t pool_id, size_t & bytes) { + fixed_pool &self = *reinterpret_cast(pool_id); + __TBBMALLOC_ASSERT(0 != self.my_size, "The buffer must not be used twice."); + bytes = self.my_size; + self.my_size = 0; // remember that buffer has been used + return self.my_buffer; +} + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::memory_pool_allocator; +using detail::d1::memory_pool; +using detail::d1::fixed_pool; +} // inline namepspace v1 +} // namespace tbb + +#undef __TBBMALLOC_ASSERT +#endif// __TBB_memory_pool_H diff --git a/src/3rd party/oneapi/tbb/mutex.h b/src/3rd party/oneapi/tbb/mutex.h new file mode 100644 index 0000000000..169b7a3ca9 --- /dev/null +++ b/src/3rd party/oneapi/tbb/mutex.h @@ -0,0 +1,93 @@ +/* + Copyright (c) 2021-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_mutex_H +#define __TBB_mutex_H + +#include "detail/_namespace_injection.h" +#include "detail/_utils.h" +#include "detail/_scoped_lock.h" +#include "detail/_waitable_atomic.h" +#include "detail/_mutex_common.h" +#include "profiling.h" + +namespace tbb { +namespace detail { +namespace d1 { + +class mutex { +public: + //! Constructors + mutex() { + create_itt_sync(this, "tbb::mutex", ""); + }; + + //! Destructor + ~mutex() = default; + + //! No Copy + mutex(const mutex&) = delete; + mutex& operator=(const mutex&) = delete; + + using scoped_lock = unique_scoped_lock; + + //! Mutex traits + static constexpr bool is_rw_mutex = false; + static constexpr bool is_recursive_mutex = false; + static constexpr bool is_fair_mutex = false; + + //! Acquire lock + /** Spin if the lock is taken */ + void lock() { + call_itt_notify(prepare, this); + while (!try_lock()) { + my_flag.wait(true, /* context = */ 0, std::memory_order_relaxed); + } + } + + //! Try acquiring lock (non-blocking) + /** Return true if lock acquired; false otherwise. */ + bool try_lock() { + bool result = !my_flag.load(std::memory_order_relaxed) && !my_flag.exchange(true); + if (result) { + call_itt_notify(acquired, this); + } + return result; + } + + //! Release lock + void unlock() { + call_itt_notify(releasing, this); + // We need Write Read memory barrier before notify that reads the waiter list. + // In C++ only full fence covers this type of barrier. + my_flag.exchange(false); + my_flag.notify_one_relaxed(); + } + +private: + waitable_atomic my_flag{0}; +}; // class mutex + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::mutex; +} // namespace v1 + +} // namespace tbb + +#endif // __TBB_mutex_H diff --git a/src/3rd party/oneapi/tbb/null_mutex.h b/src/3rd party/oneapi/tbb/null_mutex.h new file mode 100644 index 0000000000..1797e35bb7 --- /dev/null +++ b/src/3rd party/oneapi/tbb/null_mutex.h @@ -0,0 +1,80 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_null_mutex_H +#define __TBB_null_mutex_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_mutex_common.h" + +namespace tbb { +namespace detail { +namespace d1 { + +//! A mutex which does nothing +/** A null_mutex does no operation and simulates success. + @ingroup synchronization */ +class null_mutex { +public: + //! Constructors + constexpr null_mutex() noexcept = default; + + //! Destructor + ~null_mutex() = default; + + //! No Copy + null_mutex(const null_mutex&) = delete; + null_mutex& operator=(const null_mutex&) = delete; + + //! Represents acquisition of a mutex. + class scoped_lock { + public: + //! Constructors + constexpr scoped_lock() noexcept = default; + scoped_lock(null_mutex&) {} + + //! Destructor + ~scoped_lock() = default; + + //! No Copy + scoped_lock(const scoped_lock&) = delete; + scoped_lock& operator=(const scoped_lock&) = delete; + + void acquire(null_mutex&) {} + bool try_acquire(null_mutex&) { return true; } + void release() {} + }; + + //! Mutex traits + static constexpr bool is_rw_mutex = false; + static constexpr bool is_recursive_mutex = true; + static constexpr bool is_fair_mutex = true; + + void lock() {} + bool try_lock() { return true; } + void unlock() {} +}; // class null_mutex + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::null_mutex; +} // namespace v1 +} // namespace tbb + +#endif /* __TBB_null_mutex_H */ diff --git a/src/3rd party/oneapi/tbb/null_rw_mutex.h b/src/3rd party/oneapi/tbb/null_rw_mutex.h new file mode 100644 index 0000000000..43983a0058 --- /dev/null +++ b/src/3rd party/oneapi/tbb/null_rw_mutex.h @@ -0,0 +1,87 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_null_rw_mutex_H +#define __TBB_null_rw_mutex_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_mutex_common.h" + +namespace tbb { +namespace detail { +namespace d1 { + +//! A rw mutex which does nothing +/** A null_rw_mutex is a rw mutex that does nothing and simulates successful operation. + @ingroup synchronization */ +class null_rw_mutex { +public: + //! Constructors + constexpr null_rw_mutex() noexcept = default; + + //! Destructor + ~null_rw_mutex() = default; + + //! No Copy + null_rw_mutex(const null_rw_mutex&) = delete; + null_rw_mutex& operator=(const null_rw_mutex&) = delete; + + //! Represents acquisition of a mutex. + class scoped_lock { + public: + //! Constructors + constexpr scoped_lock() noexcept = default; + scoped_lock(null_rw_mutex&, bool = true) {} + + //! Destructor + ~scoped_lock() = default; + + //! No Copy + scoped_lock(const scoped_lock&) = delete; + scoped_lock& operator=(const scoped_lock&) = delete; + + void acquire(null_rw_mutex&, bool = true) {} + bool try_acquire(null_rw_mutex&, bool = true) { return true; } + void release() {} + bool upgrade_to_writer() { return true; } + bool downgrade_to_reader() { return true; } + + bool is_writer() const { return true; } + }; + + //! Mutex traits + static constexpr bool is_rw_mutex = true; + static constexpr bool is_recursive_mutex = true; + static constexpr bool is_fair_mutex = true; + + void lock() {} + bool try_lock() { return true; } + void unlock() {} + void lock_shared() {} + bool try_lock_shared() { return true; } + void unlock_shared() {} +}; // class null_rw_mutex + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::null_rw_mutex; +} // namespace v1 +} // namespace tbb + +#endif /* __TBB_null_rw_mutex_H */ diff --git a/src/3rd party/oneapi/tbb/parallel_for.h b/src/3rd party/oneapi/tbb/parallel_for.h new file mode 100644 index 0000000000..37a2613508 --- /dev/null +++ b/src/3rd party/oneapi/tbb/parallel_for.h @@ -0,0 +1,469 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_parallel_for_H +#define __TBB_parallel_for_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_exception.h" +#include "detail/_task.h" +#include "detail/_small_object_pool.h" +#include "profiling.h" + +#include "partitioner.h" +#include "blocked_range.h" +#include "task_group.h" + +#include +#include + +namespace tbb { +namespace detail { +#if __TBB_CPP20_CONCEPTS_PRESENT +inline namespace d0 { + +template +concept parallel_for_body = std::copy_constructible && std::invocable&, Range&>; + +template +concept parallel_for_index = std::constructible_from && + std::copyable && + requires( const std::remove_reference_t& lhs, const std::remove_reference_t& rhs ) { + { lhs < rhs } -> adaptive_same_as; + { lhs - rhs } -> std::convertible_to; + { lhs + (rhs - lhs) } -> std::convertible_to; + }; + +template +concept parallel_for_function = std::invocable&, Index>; + +} // namespace d0 +#endif // __TBB_CPP20_CONCEPTS_PRESENT +namespace d1 { + +//! Task type used in parallel_for +/** @ingroup algorithms */ +template +struct start_for : public task { + Range my_range; + const Body my_body; + node* my_parent; + + typename Partitioner::task_partition_type my_partition; + small_object_allocator my_allocator; + + task* execute(execution_data&) override; + task* cancel(execution_data&) override; + void finalize(const execution_data&); + + //! Constructor for root task. + start_for( const Range& range, const Body& body, Partitioner& partitioner, small_object_allocator& alloc ) : + my_range(range), + my_body(body), + my_parent(nullptr), + my_partition(partitioner), + my_allocator(alloc) {} + //! Splitting constructor used to generate children. + /** parent_ becomes left child. Newly constructed object is right child. */ + start_for( start_for& parent_, typename Partitioner::split_type& split_obj, small_object_allocator& alloc ) : + my_range(parent_.my_range, get_range_split_object(split_obj)), + my_body(parent_.my_body), + my_parent(nullptr), + my_partition(parent_.my_partition, split_obj), + my_allocator(alloc) {} + //! Construct right child from the given range as response to the demand. + /** parent_ remains left child. Newly constructed object is right child. */ + start_for( start_for& parent_, const Range& r, depth_t d, small_object_allocator& alloc ) : + my_range(r), + my_body(parent_.my_body), + my_parent(nullptr), + my_partition(parent_.my_partition, split()), + my_allocator(alloc) + { + my_partition.align_depth( d ); + } + static void run(const Range& range, const Body& body, Partitioner& partitioner) { + task_group_context context(PARALLEL_FOR); + run(range, body, partitioner, context); + } + + static void run(const Range& range, const Body& body, Partitioner& partitioner, task_group_context& context) { + if ( !range.empty() ) { + small_object_allocator alloc{}; + start_for& for_task = *alloc.new_object(range, body, partitioner, alloc); + + // defer creation of the wait node until task allocation succeeds + wait_node wn; + for_task.my_parent = &wn; + execute_and_wait(for_task, context, wn.m_wait, context); + } + } + //! Run body for range, serves as callback for partitioner + void run_body( Range &r ) { + tbb::detail::invoke(my_body, r); + } + + //! spawn right task, serves as callback for partitioner + void offer_work(typename Partitioner::split_type& split_obj, execution_data& ed) { + offer_work_impl(ed, *this, split_obj); + } + + //! spawn right task, serves as callback for partitioner + void offer_work(const Range& r, depth_t d, execution_data& ed) { + offer_work_impl(ed, *this, r, d); + } + +private: + template + void offer_work_impl(execution_data& ed, Args&&... constructor_args) { + // New right child + small_object_allocator alloc{}; + start_for& right_child = *alloc.new_object(ed, std::forward(constructor_args)..., alloc); + + // New root node as a continuation and ref count. Left and right child attach to the new parent. + right_child.my_parent = my_parent = alloc.new_object(ed, my_parent, 2, alloc); + // Spawn the right sibling + right_child.spawn_self(ed); + } + + void spawn_self(execution_data& ed) { + my_partition.spawn_task(*this, *context(ed)); + } +}; + +//! fold the tree and deallocate the task +template +void start_for::finalize(const execution_data& ed) { + // Get the current parent and allocator an object destruction + node* parent = my_parent; + auto allocator = my_allocator; + // Task execution finished - destroy it + this->~start_for(); + // Unwind the tree decrementing the parent`s reference count + + fold_tree(parent, ed); + allocator.deallocate(this, ed); + +} + +//! execute task for parallel_for +template +task* start_for::execute(execution_data& ed) { + if (!is_same_affinity(ed)) { + my_partition.note_affinity(execution_slot(ed)); + } + my_partition.check_being_stolen(*this, ed); + my_partition.execute(*this, my_range, ed); + finalize(ed); + return nullptr; +} + +//! cancel task for parallel_for +template +task* start_for::cancel(execution_data& ed) { + finalize(ed); + return nullptr; +} + +//! Calls the function with values from range [begin, end) with a step provided +template +class parallel_for_body_wrapper : detail::no_assign { + const Function &my_func; + const Index my_begin; + const Index my_step; +public: + parallel_for_body_wrapper( const Function& _func, Index& _begin, Index& _step ) + : my_func(_func), my_begin(_begin), my_step(_step) {} + + void operator()( const blocked_range& r ) const { + // A set of local variables to help the compiler with vectorization of the following loop. + Index b = r.begin(); + Index e = r.end(); + Index ms = my_step; + Index k = my_begin + b*ms; + +#if __INTEL_COMPILER +#pragma ivdep +#if __TBB_ASSERT_ON_VECTORIZATION_FAILURE +#pragma vector always assert +#endif +#endif + for ( Index i = b; i < e; ++i, k += ms ) { + tbb::detail::invoke(my_func, k); + } + } +}; + +// Requirements on Range concept are documented in blocked_range.h + +/** \page parallel_for_body_req Requirements on parallel_for body + Class \c Body implementing the concept of parallel_for body must define: + - \code Body::Body( const Body& ); \endcode Copy constructor + - \code Body::~Body(); \endcode Destructor + - \code void Body::operator()( Range& r ) const; \endcode Function call operator applying the body to range \c r. +**/ + +/** \name parallel_for + See also requirements on \ref range_req "Range" and \ref parallel_for_body_req "parallel_for Body". **/ +//@{ + +//! Parallel iteration over range with default partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body ) { + start_for::run(range,body,__TBB_DEFAULT_PARTITIONER()); +} + +//! Parallel iteration over range with simple partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, const simple_partitioner& partitioner ) { + start_for::run(range,body,partitioner); +} + +//! Parallel iteration over range with auto_partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, const auto_partitioner& partitioner ) { + start_for::run(range,body,partitioner); +} + +//! Parallel iteration over range with static_partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, const static_partitioner& partitioner ) { + start_for::run(range,body,partitioner); +} + +//! Parallel iteration over range with affinity_partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, affinity_partitioner& partitioner ) { + start_for::run(range,body,partitioner); +} + +//! Parallel iteration over range with default partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, task_group_context& context ) { + start_for::run(range, body, __TBB_DEFAULT_PARTITIONER(), context); +} + +//! Parallel iteration over range with simple partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, const simple_partitioner& partitioner, task_group_context& context ) { + start_for::run(range, body, partitioner, context); +} + +//! Parallel iteration over range with auto_partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, const auto_partitioner& partitioner, task_group_context& context ) { + start_for::run(range, body, partitioner, context); +} + +//! Parallel iteration over range with static_partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, const static_partitioner& partitioner, task_group_context& context ) { + start_for::run(range, body, partitioner, context); +} + +//! Parallel iteration over range with affinity_partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_for_body) +void parallel_for( const Range& range, const Body& body, affinity_partitioner& partitioner, task_group_context& context ) { + start_for::run(range,body,partitioner, context); +} + +//! Implementation of parallel iteration over stepped range of integers with explicit step and partitioner +template +void parallel_for_impl(Index first, Index last, Index step, const Function& f, Partitioner& partitioner) { + if (step <= 0 ) + throw_exception(exception_id::nonpositive_step); // throws std::invalid_argument + else if (first < last) { + // Above "else" avoids "potential divide by zero" warning on some platforms + Index end = Index(last - first - 1ul) / step + Index(1); + blocked_range range(static_cast(0), end); + parallel_for_body_wrapper body(f, first, step); + parallel_for(range, body, partitioner); + } +} + +//! Parallel iteration over a range of integers with a step provided and default partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f) { + parallel_for_impl(first, last, step, f, __TBB_DEFAULT_PARTITIONER()); +} +//! Parallel iteration over a range of integers with a step provided and simple partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, const simple_partitioner& partitioner) { + parallel_for_impl(first, last, step, f, partitioner); +} +//! Parallel iteration over a range of integers with a step provided and auto partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, const auto_partitioner& partitioner) { + parallel_for_impl(first, last, step, f, partitioner); +} +//! Parallel iteration over a range of integers with a step provided and static partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, const static_partitioner& partitioner) { + parallel_for_impl(first, last, step, f, partitioner); +} +//! Parallel iteration over a range of integers with a step provided and affinity partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, affinity_partitioner& partitioner) { + parallel_for_impl(first, last, step, f, partitioner); +} + +//! Parallel iteration over a range of integers with a default step value and default partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f) { + parallel_for_impl(first, last, static_cast(1), f, __TBB_DEFAULT_PARTITIONER()); +} +//! Parallel iteration over a range of integers with a default step value and simple partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, const simple_partitioner& partitioner) { + parallel_for_impl(first, last, static_cast(1), f, partitioner); +} +//! Parallel iteration over a range of integers with a default step value and auto partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, const auto_partitioner& partitioner) { + parallel_for_impl(first, last, static_cast(1), f, partitioner); +} +//! Parallel iteration over a range of integers with a default step value and static partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, const static_partitioner& partitioner) { + parallel_for_impl(first, last, static_cast(1), f, partitioner); +} +//! Parallel iteration over a range of integers with a default step value and affinity partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, affinity_partitioner& partitioner) { + parallel_for_impl(first, last, static_cast(1), f, partitioner); +} + +//! Implementation of parallel iteration over stepped range of integers with explicit step, task group context, and partitioner +template +void parallel_for_impl(Index first, Index last, Index step, const Function& f, Partitioner& partitioner, task_group_context &context) { + if (step <= 0 ) + throw_exception(exception_id::nonpositive_step); // throws std::invalid_argument + else if (first < last) { + // Above "else" avoids "potential divide by zero" warning on some platforms + Index end = (last - first - Index(1)) / step + Index(1); + blocked_range range(static_cast(0), end); + parallel_for_body_wrapper body(f, first, step); + parallel_for(range, body, partitioner, context); + } +} + +//! Parallel iteration over a range of integers with explicit step, task group context, and default partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, task_group_context &context) { + parallel_for_impl(first, last, step, f, __TBB_DEFAULT_PARTITIONER(), context); +} +//! Parallel iteration over a range of integers with explicit step, task group context, and simple partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, const simple_partitioner& partitioner, task_group_context &context) { + parallel_for_impl(first, last, step, f, partitioner, context); +} +//! Parallel iteration over a range of integers with explicit step, task group context, and auto partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, const auto_partitioner& partitioner, task_group_context &context) { + parallel_for_impl(first, last, step, f, partitioner, context); +} +//! Parallel iteration over a range of integers with explicit step, task group context, and static partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, const static_partitioner& partitioner, task_group_context &context) { + parallel_for_impl(first, last, step, f, partitioner, context); +} +//! Parallel iteration over a range of integers with explicit step, task group context, and affinity partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, Index step, const Function& f, affinity_partitioner& partitioner, task_group_context &context) { + parallel_for_impl(first, last, step, f, partitioner, context); +} + +//! Parallel iteration over a range of integers with a default step value, explicit task group context, and default partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, task_group_context &context) { + parallel_for_impl(first, last, static_cast(1), f, __TBB_DEFAULT_PARTITIONER(), context); +} +//! Parallel iteration over a range of integers with a default step value, explicit task group context, and simple partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, const simple_partitioner& partitioner, task_group_context &context) { + parallel_for_impl(first, last, static_cast(1), f, partitioner, context); +} +//! Parallel iteration over a range of integers with a default step value, explicit task group context, and auto partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, const auto_partitioner& partitioner, task_group_context &context) { + parallel_for_impl(first, last, static_cast(1), f, partitioner, context); +} +//! Parallel iteration over a range of integers with a default step value, explicit task group context, and static partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, const static_partitioner& partitioner, task_group_context &context) { + parallel_for_impl(first, last, static_cast(1), f, partitioner, context); +} +//! Parallel iteration over a range of integers with a default step value, explicit task group context, and affinity_partitioner +template + __TBB_requires(parallel_for_index && parallel_for_function) +void parallel_for(Index first, Index last, const Function& f, affinity_partitioner& partitioner, task_group_context &context) { + parallel_for_impl(first, last, static_cast(1), f, partitioner, context); +} +// @} + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::parallel_for; +// Split types +using detail::split; +using detail::proportional_split; +} // namespace v1 + +} // namespace tbb + +#endif /* __TBB_parallel_for_H */ diff --git a/src/3rd party/oneapi/tbb/parallel_for_each.h b/src/3rd party/oneapi/tbb/parallel_for_each.h new file mode 100644 index 0000000000..b64eb233ec --- /dev/null +++ b/src/3rd party/oneapi/tbb/parallel_for_each.h @@ -0,0 +1,712 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_parallel_for_each_H +#define __TBB_parallel_for_each_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_exception.h" +#include "detail/_task.h" +#include "detail/_aligned_space.h" +#include "detail/_small_object_pool.h" +#include "detail/_utils.h" + +#include "parallel_for.h" +#include "task_group.h" // task_group_context + +#include +#include + +namespace tbb { +namespace detail { +#if __TBB_CPP20_CONCEPTS_PRESENT +namespace d1 { +template +class feeder; + +} // namespace d1 +inline namespace d0 { + +template +concept parallel_for_each_body = std::invocable&, ItemType&&> || + std::invocable&, ItemType&&, tbb::detail::d1::feeder&>; + +} // namespace d0 +#endif // __TBB_CPP20_CONCEPTS_PRESENT +namespace d2 { +template class feeder_impl; +} // namespace d2 + +namespace d1 { +//! Class the user supplied algorithm body uses to add new tasks +template +class feeder { + feeder() {} + feeder(const feeder&) = delete; + void operator=( const feeder&) = delete; + + virtual ~feeder () {} + virtual void internal_add_copy(const Item& item) = 0; + virtual void internal_add_move(Item&& item) = 0; + + template friend class d2::feeder_impl; +public: + //! Add a work item to a running parallel_for_each. + void add(const Item& item) {internal_add_copy(item);} + void add(Item&& item) {internal_add_move(std::move(item));} +}; + +} // namespace d1 + +namespace d2 { +using namespace tbb::detail::d1; +/** Selects one of the two possible forms of function call member operator. + @ingroup algorithms **/ +template +struct parallel_for_each_operator_selector { +public: + template + static auto call(const Body& body, ItemArg&& item, FeederArg*) + -> decltype(tbb::detail::invoke(body, std::forward(item)), void()) { + #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + // Suppression of Microsoft non-standard extension warnings + #pragma warning (push) + #pragma warning (disable: 4239) + #endif + + tbb::detail::invoke(body, std::forward(item)); + + #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + #pragma warning (pop) + #endif + } + + template + static auto call(const Body& body, ItemArg&& item, FeederArg* feeder) + -> decltype(tbb::detail::invoke(body, std::forward(item), *feeder), void()) { + #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + // Suppression of Microsoft non-standard extension warnings + #pragma warning (push) + #pragma warning (disable: 4239) + #endif + __TBB_ASSERT(feeder, "Feeder was not created but should be"); + + tbb::detail::invoke(body, std::forward(item), *feeder); + + #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + #pragma warning (pop) + #endif + } +}; + +template +struct feeder_item_task: public task { + using feeder_type = feeder_impl; + + template + feeder_item_task(ItemType&& input_item, feeder_type& feeder, small_object_allocator& alloc, wait_tree_vertex_interface& wait_vertex) : + item(std::forward(input_item)), + my_feeder(feeder), + my_allocator(alloc), + m_wait_tree_vertex(r1::get_thread_reference_vertex(&wait_vertex)) + { + m_wait_tree_vertex->reserve(); + } + + void finalize(const execution_data& ed) { + m_wait_tree_vertex->release(); + my_allocator.delete_object(this, ed); + } + + //! Hack for resolve ambiguity between calls to the body with and without moving the stored copy + //! Executing body with moving the copy should have higher priority + using first_priority = int; + using second_priority = double; + + template + static auto call(const BodyType& call_body, ItemType& call_item, FeederType& call_feeder, first_priority) + -> decltype(parallel_for_each_operator_selector::call(call_body, std::move(call_item), &call_feeder), void()) + { + parallel_for_each_operator_selector::call(call_body, std::move(call_item), &call_feeder); + } + + template + static void call(const BodyType& call_body, ItemType& call_item, FeederType& call_feeder, second_priority) { + parallel_for_each_operator_selector::call(call_body, call_item, &call_feeder); + } + + task* execute(execution_data& ed) override { + call(my_feeder.my_body, item, my_feeder, first_priority{}); + finalize(ed); + return nullptr; + } + + task* cancel(execution_data& ed) override { + finalize(ed); + return nullptr; + } + + Item item; + feeder_type& my_feeder; + small_object_allocator my_allocator; + wait_tree_vertex_interface* m_wait_tree_vertex; +}; // class feeder_item_task + +/** Implements new task adding procedure. + @ingroup algorithms **/ +template +class feeder_impl : public feeder { + // Avoiding use of copy constructor in a virtual method if the type does not support it + void internal_add_copy_impl(std::true_type, const Item& item) { + using feeder_task = feeder_item_task; + small_object_allocator alloc; + auto task = alloc.new_object(item, *this, alloc, my_wait_context); + + spawn(*task, my_execution_context); + } + + void internal_add_copy_impl(std::false_type, const Item&) { + __TBB_ASSERT(false, "Overloading for r-value reference doesn't work or it's not movable and not copyable object"); + } + + void internal_add_copy(const Item& item) override { + internal_add_copy_impl(typename std::is_copy_constructible::type(), item); + } + + void internal_add_move(Item&& item) override { + using feeder_task = feeder_item_task; + small_object_allocator alloc{}; + auto task = alloc.new_object(std::move(item), *this, alloc, my_wait_context); + + spawn(*task, my_execution_context); + } +public: + feeder_impl(const Body& body, wait_context_vertex& w_context, task_group_context &context) + : my_body(body), + my_wait_context(w_context) + , my_execution_context(context) + {} + + const Body& my_body; + wait_context_vertex& my_wait_context; + task_group_context& my_execution_context; +}; // class feeder_impl + +/** Execute computation under one element of the range + @ingroup algorithms **/ +template +struct for_each_iteration_task: public task { + using feeder_type = feeder_impl; + + for_each_iteration_task(Iterator input_item_ptr, const Body& body, feeder_impl* feeder_ptr, wait_context& wait_context) : + item_ptr(input_item_ptr), my_body(body), my_feeder_ptr(feeder_ptr), parent_wait_context(wait_context) + {} + + void finalize() { + parent_wait_context.release(); + } + + task* execute(execution_data&) override { + parallel_for_each_operator_selector::call(my_body, *item_ptr, my_feeder_ptr); + finalize(); + return nullptr; + } + + task* cancel(execution_data&) override { + finalize(); + return nullptr; + } + + Iterator item_ptr; + const Body& my_body; + feeder_impl* my_feeder_ptr; + wait_context& parent_wait_context; +}; // class for_each_iteration_task + +// Helper to get the type of the iterator to the internal sequence of copies +// If the element can be passed to the body as an rvalue - this iterator should be move_iterator +template +struct input_iteration_task_iterator_helper { + // For input iterators we pass const lvalue reference to the body + // It is prohibited to take non-constant lvalue references for input iterators + using type = const Item*; +}; + +template +struct input_iteration_task_iterator_helper::call(std::declval(), + std::declval(), + std::declval*>()))>> +{ + using type = std::move_iterator; +}; + +/** Split one block task to several(max_block_size) iteration tasks for input iterators + @ingroup algorithms **/ +template +struct input_block_handling_task : public task { + static constexpr size_t max_block_size = 4; + + using feeder_type = feeder_impl; + using iteration_task_iterator_type = typename input_iteration_task_iterator_helper::type; + using iteration_task = for_each_iteration_task; + + input_block_handling_task(wait_context_vertex& root_wait_context, task_group_context& e_context, + const Body& body, feeder_impl* feeder_ptr, small_object_allocator& alloc) + :my_size(0), my_wait_context(0), my_root_wait_context(root_wait_context), + my_execution_context(e_context), my_allocator(alloc) + { + auto item_it = block_iteration_space.begin(); + for (auto* it = task_pool.begin(); it != task_pool.end(); ++it) { + new (it) iteration_task(iteration_task_iterator_type(item_it++), body, feeder_ptr, my_wait_context); + } + } + + void finalize(const execution_data& ed) { + my_root_wait_context.release(); + my_allocator.delete_object(this, ed); + } + + task* execute(execution_data& ed) override { + __TBB_ASSERT( my_size > 0, "Negative size was passed to task"); + for (std::size_t counter = 1; counter < my_size; ++counter) { + my_wait_context.reserve(); + spawn(*(task_pool.begin() + counter), my_execution_context); + } + my_wait_context.reserve(); + execute_and_wait(*task_pool.begin(), my_execution_context, + my_wait_context, my_execution_context); + + // deallocate current task after children execution + finalize(ed); + return nullptr; + } + + task* cancel(execution_data& ed) override { + finalize(ed); + return nullptr; + } + + ~input_block_handling_task() { + for(std::size_t counter = 0; counter < max_block_size; ++counter) { + (task_pool.begin() + counter)->~iteration_task(); + if (counter < my_size) { + (block_iteration_space.begin() + counter)->~Item(); + } + } + } + + aligned_space block_iteration_space; + aligned_space task_pool; + std::size_t my_size; + wait_context my_wait_context; + wait_context_vertex& my_root_wait_context; + task_group_context& my_execution_context; + small_object_allocator my_allocator; +}; // class input_block_handling_task + +/** Split one block task to several(max_block_size) iteration tasks for forward iterators + @ingroup algorithms **/ +template +struct forward_block_handling_task : public task { + static constexpr size_t max_block_size = 4; + + using iteration_task = for_each_iteration_task; + + forward_block_handling_task(Iterator first, std::size_t size, + wait_context_vertex& w_context, task_group_context& e_context, + const Body& body, feeder_impl* feeder_ptr, + small_object_allocator& alloc) + : my_size(size), my_wait_context(0), my_root_wait_context(w_context), + my_execution_context(e_context), my_allocator(alloc) + { + auto* task_it = task_pool.begin(); + for (std::size_t i = 0; i < size; i++) { + new (task_it++) iteration_task(first, body, feeder_ptr, my_wait_context); + ++first; + } + } + + void finalize(const execution_data& ed) { + my_root_wait_context.release(); + my_allocator.delete_object(this, ed); + } + + task* execute(execution_data& ed) override { + __TBB_ASSERT( my_size > 0, "Negative size was passed to task"); + for(std::size_t counter = 1; counter < my_size; ++counter) { + my_wait_context.reserve(); + spawn(*(task_pool.begin() + counter), my_execution_context); + } + my_wait_context.reserve(); + execute_and_wait(*task_pool.begin(), my_execution_context, + my_wait_context, my_execution_context); + + // deallocate current task after children execution + finalize(ed); + return nullptr; + } + + task* cancel(execution_data& ed) override { + finalize(ed); + return nullptr; + } + + ~forward_block_handling_task() { + for(std::size_t counter = 0; counter < my_size; ++counter) { + (task_pool.begin() + counter)->~iteration_task(); + } + } + + aligned_space task_pool; + std::size_t my_size; + wait_context my_wait_context; + wait_context_vertex& my_root_wait_context; + task_group_context& my_execution_context; + small_object_allocator my_allocator; +}; // class forward_block_handling_task + +/** Body for parallel_for algorithm. + * Allows to redirect operations under random access iterators range to the parallel_for algorithm. + @ingroup algorithms **/ +template +class parallel_for_body_wrapper { + Iterator my_first; + const Body& my_body; + feeder_impl* my_feeder_ptr; +public: + parallel_for_body_wrapper(Iterator first, const Body& body, feeder_impl* feeder_ptr) + : my_first(first), my_body(body), my_feeder_ptr(feeder_ptr) {} + + void operator()(tbb::blocked_range range) const { +#if __INTEL_COMPILER +#pragma ivdep +#endif + for (std::size_t count = range.begin(); count != range.end(); count++) { + parallel_for_each_operator_selector::call(my_body, *(my_first + count), + my_feeder_ptr); + } + } +}; // class parallel_for_body_wrapper + + +/** Helper for getting iterators tag including inherited custom tags + @ingroup algorithms */ +template +using tag = typename std::iterator_traits::iterator_category; + +#if __TBB_CPP20_CONCEPTS_PRESENT +template +struct move_iterator_dispatch_helper { + using type = It; +}; + +// Until C++23, std::move_iterator::iterator_concept always defines +// to std::input_iterator_tag and hence std::forward_iterator concept +// always evaluates to false, so std::move_iterator dispatch should be +// made according to the base iterator type. +template +struct move_iterator_dispatch_helper> { + using type = It; +}; + +template +using iterator_tag_dispatch_impl = + std::conditional_t, + std::random_access_iterator_tag, + std::conditional_t, + std::forward_iterator_tag, + std::input_iterator_tag>>; + +template +using iterator_tag_dispatch = + iterator_tag_dispatch_impl::type>; + +#else +template +using iterator_tag_dispatch = typename + std::conditional< + std::is_base_of>::value, + std::random_access_iterator_tag, + typename std::conditional< + std::is_base_of>::value, + std::forward_iterator_tag, + std::input_iterator_tag + >::type + >::type; +#endif // __TBB_CPP20_CONCEPTS_PRESENT + +template +using feeder_is_required = tbb::detail::void_t(), + std::declval::reference>(), + std::declval&>()))>; + +// Creates feeder object only if the body can accept it +template +struct feeder_holder { + feeder_holder( wait_context_vertex&, task_group_context&, const Body& ) {} + + feeder_impl* feeder_ptr() { return nullptr; } +}; // class feeder_holder + +template +class feeder_holder> { +public: + feeder_holder( wait_context_vertex& w_context, task_group_context& context, const Body& body ) + : my_feeder(body, w_context, context) {} + + feeder_impl* feeder_ptr() { return &my_feeder; } +private: + feeder_impl my_feeder; +}; // class feeder_holder + +template +class for_each_root_task_base : public task { +public: + for_each_root_task_base(Iterator first, Iterator last, const Body& body, wait_context_vertex& w_context, task_group_context& e_context) + : my_first(first), my_last(last), my_wait_context(w_context), my_execution_context(e_context), + my_body(body), my_feeder_holder(my_wait_context, my_execution_context, my_body) + { + my_wait_context.reserve(); + } +private: + task* cancel(execution_data&) override { + this->my_wait_context.release(); + return nullptr; + } +protected: + Iterator my_first; + Iterator my_last; + wait_context_vertex& my_wait_context; + task_group_context& my_execution_context; + const Body& my_body; + feeder_holder my_feeder_holder; +}; // class for_each_root_task_base + +/** parallel_for_each algorithm root task - most generic version + * Splits input range to blocks + @ingroup algorithms **/ +template > +class for_each_root_task : public for_each_root_task_base +{ + using base_type = for_each_root_task_base; +public: + using base_type::base_type; +private: + task* execute(execution_data& ed) override { + using block_handling_type = input_block_handling_task; + + if (this->my_first == this->my_last) { + this->my_wait_context.release(); + return nullptr; + } + + this->my_wait_context.reserve(); + small_object_allocator alloc{}; + auto block_handling_task = alloc.new_object(ed, this->my_wait_context, this->my_execution_context, + this->my_body, this->my_feeder_holder.feeder_ptr(), + alloc); + + auto* block_iterator = block_handling_task->block_iteration_space.begin(); + for (; !(this->my_first == this->my_last) && block_handling_task->my_size < block_handling_type::max_block_size; ++this->my_first) { + // Move semantics are automatically used when supported by the iterator + new (block_iterator++) Item(*this->my_first); + ++block_handling_task->my_size; + } + + // Do not access this after spawn to avoid races + spawn(*this, this->my_execution_context); + return block_handling_task; + } +}; // class for_each_root_task - most generic implementation + +/** parallel_for_each algorithm root task - forward iterator based specialization + * Splits input range to blocks + @ingroup algorithms **/ +template +class for_each_root_task + : public for_each_root_task_base +{ + using base_type = for_each_root_task_base; +public: + using base_type::base_type; +private: + task* execute(execution_data& ed) override { + using block_handling_type = forward_block_handling_task; + if (this->my_first == this->my_last) { + this->my_wait_context.release(); + return nullptr; + } + + std::size_t block_size{0}; + Iterator first_block_element = this->my_first; + for (; !(this->my_first == this->my_last) && block_size < block_handling_type::max_block_size; ++this->my_first) { + ++block_size; + } + + this->my_wait_context.reserve(); + small_object_allocator alloc{}; + auto block_handling_task = alloc.new_object(ed, first_block_element, block_size, + this->my_wait_context, this->my_execution_context, + this->my_body, this->my_feeder_holder.feeder_ptr(), alloc); + + // Do not access this after spawn to avoid races + spawn(*this, this->my_execution_context); + return block_handling_task; + } +}; // class for_each_root_task - forward iterator based specialization + +/** parallel_for_each algorithm root task - random access iterator based specialization + * Splits input range to blocks + @ingroup algorithms **/ +template +class for_each_root_task + : public for_each_root_task_base +{ + using base_type = for_each_root_task_base; +public: + using base_type::base_type; +private: + task* execute(execution_data&) override { + tbb::parallel_for( + tbb::blocked_range(0, std::distance(this->my_first, this->my_last)), + parallel_for_body_wrapper(this->my_first, this->my_body, this->my_feeder_holder.feeder_ptr()) + , this->my_execution_context + ); + + this->my_wait_context.release(); + return nullptr; + } +}; // class for_each_root_task - random access iterator based specialization + +/** Helper for getting item type. If item type can be deduced from feeder - got it from feeder, + if feeder is generic - got item type from range. + @ingroup algorithms */ +template +auto feeder_argument_parser(void (Body::*)(Item, feeder&) const) -> FeederArg; + +template +decltype(feeder_argument_parser(&Body::operator())) get_item_type_impl(int); // for (T, feeder) +template Item get_item_type_impl(...); // stub + +template +using get_item_type = decltype(get_item_type_impl(0)); + +#if __TBB_CPP20_CONCEPTS_PRESENT +template +using feeder_item_type = std::remove_cvref_t>; + +template +concept parallel_for_each_iterator_body = + parallel_for_each_body, feeder_item_type>>; + +template +concept parallel_for_each_range_body = + parallel_for_each_body, feeder_item_type>>; +#endif + +/** Implements parallel iteration over a range. + @ingroup algorithms */ +template +void run_parallel_for_each( Iterator first, Iterator last, const Body& body, task_group_context& context) +{ + if (!(first == last)) { + using ItemType = get_item_type::value_type>; + wait_context_vertex w_context(0); + + for_each_root_task root_task(first, last, body, w_context, context); + + execute_and_wait(root_task, context, w_context.get_context(), context); + } +} + +/** \page parallel_for_each_body_req Requirements on parallel_for_each body + Class \c Body implementing the concept of parallel_for_each body must define: + - \code + B::operator()( + cv_item_type item, + feeder& feeder + ) const + + OR + + B::operator()( cv_item_type& item ) const + \endcode Process item. + May be invoked concurrently for the same \c this but different \c item. + + - \code item_type( const item_type& ) \endcode + Copy a work item. + - \code ~item_type() \endcode Destroy a work item +**/ + +/** \name parallel_for_each + See also requirements on \ref parallel_for_each_body_req "parallel_for_each Body". **/ +//@{ +//! Parallel iteration over a range, with optional addition of more work. +/** @ingroup algorithms */ +template + __TBB_requires(std::input_iterator && parallel_for_each_iterator_body) +void parallel_for_each(Iterator first, Iterator last, const Body& body) { + task_group_context context(PARALLEL_FOR_EACH); + run_parallel_for_each(first, last, body, context); +} + +template + __TBB_requires(container_based_sequence && parallel_for_each_range_body) +void parallel_for_each(Range& rng, const Body& body) { + parallel_for_each(std::begin(rng), std::end(rng), body); +} + +template + __TBB_requires(container_based_sequence && parallel_for_each_range_body) +void parallel_for_each(const Range& rng, const Body& body) { + parallel_for_each(std::begin(rng), std::end(rng), body); +} + +//! Parallel iteration over a range, with optional addition of more work and user-supplied context +/** @ingroup algorithms */ +template + __TBB_requires(std::input_iterator && parallel_for_each_iterator_body) +void parallel_for_each(Iterator first, Iterator last, const Body& body, task_group_context& context) { + run_parallel_for_each(first, last, body, context); +} + +template + __TBB_requires(container_based_sequence && parallel_for_each_range_body) +void parallel_for_each(Range& rng, const Body& body, task_group_context& context) { + parallel_for_each(std::begin(rng), std::end(rng), body, context); +} + +template + __TBB_requires(container_based_sequence && parallel_for_each_range_body) +void parallel_for_each(const Range& rng, const Body& body, task_group_context& context) { + parallel_for_each(std::begin(rng), std::end(rng), body, context); +} + +} // namespace d2 +} // namespace detail +//! @endcond +//@} + +inline namespace v1 { +using detail::d2::parallel_for_each; +using detail::d1::feeder; +} // namespace v1 + +} // namespace tbb + +#endif /* __TBB_parallel_for_each_H */ diff --git a/src/3rd party/oneapi/tbb/parallel_invoke.h b/src/3rd party/oneapi/tbb/parallel_invoke.h new file mode 100644 index 0000000000..4bc5d85339 --- /dev/null +++ b/src/3rd party/oneapi/tbb/parallel_invoke.h @@ -0,0 +1,227 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_parallel_invoke_H +#define __TBB_parallel_invoke_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_exception.h" +#include "detail/_task.h" +#include "detail/_template_helpers.h" +#include "detail/_small_object_pool.h" + +#include "task_group.h" + +#include +#include +#include + +namespace tbb { +namespace detail { +namespace d1 { + +//! Simple task object, executing user method +template +struct function_invoker : public task { + function_invoker(const Function& function, WaitObject& wait_ctx) : + my_function(function), + parent_wait_ctx(wait_ctx) + {} + + task* execute(execution_data& ed) override { + my_function(); + parent_wait_ctx.release(ed); + call_itt_task_notify(destroy, this); + return nullptr; + } + + task* cancel(execution_data& ed) override { + parent_wait_ctx.release(ed); + return nullptr; + } + + const Function& my_function; + WaitObject& parent_wait_ctx; +}; // struct function_invoker + +//! Task object for managing subroots in trinary task trees. +// Endowed with additional synchronization logic (compatible with wait object interfaces) to support +// continuation passing execution. This task spawns 2 function_invoker tasks with first and second functors +// and then executes first functor by itself. But only the last executed functor must destruct and deallocate +// the subroot task. +template +struct invoke_subroot_task : public task { + wait_context& root_wait_ctx; + std::atomic ref_count{0}; + bool child_spawned = false; + + const F1& self_invoked_functor; + function_invoker> f2_invoker; + function_invoker> f3_invoker; + + task_group_context& my_execution_context; + small_object_allocator my_allocator; + + invoke_subroot_task(const F1& f1, const F2& f2, const F3& f3, wait_context& wait_ctx, task_group_context& context, + small_object_allocator& alloc) : + root_wait_ctx(wait_ctx), + self_invoked_functor(f1), + f2_invoker(f2, *this), + f3_invoker(f3, *this), + my_execution_context(context), + my_allocator(alloc) + { + root_wait_ctx.reserve(); + } + + void finalize(const execution_data& ed) { + root_wait_ctx.release(); + + my_allocator.delete_object(this, ed); + } + + void release(const execution_data& ed) { + __TBB_ASSERT(ref_count > 0, nullptr); + call_itt_task_notify(releasing, this); + if( --ref_count == 0 ) { + call_itt_task_notify(acquired, this); + finalize(ed); + } + } + + task* execute(execution_data& ed) override { + ref_count.fetch_add(3, std::memory_order_relaxed); + spawn(f3_invoker, my_execution_context); + spawn(f2_invoker, my_execution_context); + self_invoked_functor(); + + release(ed); + return nullptr; + } + + task* cancel(execution_data& ed) override { + if( ref_count > 0 ) { // detect children spawn + release(ed); + } else { + finalize(ed); + } + return nullptr; + } +}; // struct subroot_task + +class invoke_root_task { +public: + invoke_root_task(wait_context& wc) : my_wait_context(wc) {} + void release(const execution_data&) { + my_wait_context.release(); + } +private: + wait_context& my_wait_context; +}; + +template +void invoke_recursive_separation(wait_context& root_wait_ctx, task_group_context& context, const F1& f1) { + root_wait_ctx.reserve(1); + invoke_root_task root(root_wait_ctx); + function_invoker invoker1(f1, root); + + execute_and_wait(invoker1, context, root_wait_ctx, context); +} + +template +void invoke_recursive_separation(wait_context& root_wait_ctx, task_group_context& context, const F1& f1, const F2& f2) { + root_wait_ctx.reserve(2); + invoke_root_task root(root_wait_ctx); + function_invoker invoker1(f1, root); + function_invoker invoker2(f2, root); + + spawn(invoker1, context); + execute_and_wait(invoker2, context, root_wait_ctx, context); +} + +template +void invoke_recursive_separation(wait_context& root_wait_ctx, task_group_context& context, const F1& f1, const F2& f2, const F3& f3) { + root_wait_ctx.reserve(3); + invoke_root_task root(root_wait_ctx); + function_invoker invoker1(f1, root); + function_invoker invoker2(f2, root); + function_invoker invoker3(f3, root); + + //TODO: implement sub root for two tasks (measure performance) + spawn(invoker1, context); + spawn(invoker2, context); + execute_and_wait(invoker3, context, root_wait_ctx, context); +} + +template +void invoke_recursive_separation(wait_context& root_wait_ctx, task_group_context& context, + const F1& f1, const F2& f2, const F3& f3, const Fs&... fs) { + small_object_allocator alloc{}; + auto sub_root = alloc.new_object>(f1, f2, f3, root_wait_ctx, context, alloc); + spawn(*sub_root, context); + + invoke_recursive_separation(root_wait_ctx, context, fs...); +} + +template +void parallel_invoke_impl(task_group_context& context, const Fs&... fs) { + static_assert(sizeof...(Fs) >= 2, "Parallel invoke may be called with at least two callable"); + wait_context root_wait_ctx{0}; + + invoke_recursive_separation(root_wait_ctx, context, fs...); +} + +template +void parallel_invoke_impl(const F1& f1, const Fs&... fs) { + static_assert(sizeof...(Fs) >= 1, "Parallel invoke may be called with at least two callable"); + task_group_context context(PARALLEL_INVOKE); + wait_context root_wait_ctx{0}; + + invoke_recursive_separation(root_wait_ctx, context, fs..., f1); +} + +//! Passes last argument of variadic pack as first for handling user provided task_group_context +template +struct invoke_helper; + +template +struct invoke_helper, T, Fs...> : invoke_helper, Fs...> {}; + +template +struct invoke_helper, T> { + void operator()(Fs&&... args, T&& t) { + parallel_invoke_impl(std::forward(t), std::forward(args)...); + } +}; + +//! Parallel execution of several function objects +// We need to pass parameter pack through forwarding reference, +// since this pack may contain task_group_context that must be passed via lvalue non-const reference +template +void parallel_invoke(Fs&&... fs) { + invoke_helper, Fs...>()(std::forward(fs)...); +} + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::parallel_invoke; +} // namespace v1 + +} // namespace tbb +#endif /* __TBB_parallel_invoke_H */ diff --git a/src/3rd party/oneapi/tbb/parallel_pipeline.h b/src/3rd party/oneapi/tbb/parallel_pipeline.h new file mode 100644 index 0000000000..a204b9c4c6 --- /dev/null +++ b/src/3rd party/oneapi/tbb/parallel_pipeline.h @@ -0,0 +1,153 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_parallel_pipeline_H +#define __TBB_parallel_pipeline_H + +#include "detail/_pipeline_filters.h" +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "task_group.h" + +#include +#include +#include + +namespace tbb { +namespace detail { + +namespace r1 { +TBB_EXPORT void __TBB_EXPORTED_FUNC parallel_pipeline(task_group_context&, std::size_t, const d1::filter_node&); +} + +namespace d1 { + +enum class filter_mode : unsigned int +{ + //! processes multiple items in parallel and in no particular order + parallel = base_filter::filter_is_out_of_order, + //! processes items one at a time; all such filters process items in the same order + serial_in_order = base_filter::filter_is_serial, + //! processes items one at a time and in no particular order + serial_out_of_order = base_filter::filter_is_serial | base_filter::filter_is_out_of_order +}; +//! Class representing a chain of type-safe pipeline filters +/** @ingroup algorithms */ +template +class filter { + filter_node_ptr my_root; + filter( filter_node_ptr root ) : my_root(root) {} + friend void parallel_pipeline( size_t, const filter&, task_group_context& ); + template + friend filter make_filter( filter_mode, const Body& ); + template + friend filter operator&( const filter&, const filter& ); +public: + filter() = default; + filter( const filter& rhs ) : my_root(rhs.my_root) {} + filter( filter&& rhs ) : my_root(std::move(rhs.my_root)) {} + + void operator=(const filter& rhs) { + my_root = rhs.my_root; + } + void operator=( filter&& rhs ) { + my_root = std::move(rhs.my_root); + } + + template + filter( filter_mode mode, const Body& body ) : + my_root( new(r1::allocate_memory(sizeof(filter_node_leaf))) + filter_node_leaf(static_cast(mode), body) ) { + } + + filter& operator&=( const filter& right ) { + *this = *this & right; + return *this; + } + + void clear() { + // Like operator= with filter() on right side. + my_root = nullptr; + } +}; + +//! Create a filter to participate in parallel_pipeline +/** @ingroup algorithms */ +template +filter make_filter( filter_mode mode, const Body& body ) { + return filter_node_ptr( new(r1::allocate_memory(sizeof(filter_node_leaf))) + filter_node_leaf(static_cast(mode), body) ); +} + +//! Create a filter to participate in parallel_pipeline +/** @ingroup algorithms */ +template +filter, filter_output> make_filter( filter_mode mode, const Body& body ) { + return make_filter, filter_output>(mode, body); +} + +//! Composition of filters left and right. +/** @ingroup algorithms */ +template +filter operator&( const filter& left, const filter& right ) { + __TBB_ASSERT(left.my_root,"cannot use default-constructed filter as left argument of '&'"); + __TBB_ASSERT(right.my_root,"cannot use default-constructed filter as right argument of '&'"); + return filter_node_ptr( new (r1::allocate_memory(sizeof(filter_node))) filter_node(left.my_root,right.my_root) ); +} + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +template +filter(filter_mode, Body) +->filter, filter_output>; +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +//! Parallel pipeline over chain of filters with user-supplied context. +/** @ingroup algorithms **/ +inline void parallel_pipeline(size_t max_number_of_live_tokens, const filter& filter_chain, task_group_context& context) { + r1::parallel_pipeline(context, max_number_of_live_tokens, *filter_chain.my_root); +} + +//! Parallel pipeline over chain of filters. +/** @ingroup algorithms **/ +inline void parallel_pipeline(size_t max_number_of_live_tokens, const filter& filter_chain) { + task_group_context context; + parallel_pipeline(max_number_of_live_tokens, filter_chain, context); +} + +//! Parallel pipeline over sequence of filters. +/** @ingroup algorithms **/ +template +void parallel_pipeline(size_t max_number_of_live_tokens, + const F1& filter1, + const F2& filter2, + FiltersContext&&... filters) { + parallel_pipeline(max_number_of_live_tokens, filter1 & filter2, std::forward(filters)...); +} + +} // namespace d1 +} // namespace detail + +inline namespace v1 +{ +using detail::d1::parallel_pipeline; +using detail::d1::filter; +using detail::d1::make_filter; +using detail::d1::filter_mode; +using detail::d1::flow_control; +} +} // tbb + +#endif /* __TBB_parallel_pipeline_H */ diff --git a/src/3rd party/oneapi/tbb/parallel_reduce.h b/src/3rd party/oneapi/tbb/parallel_reduce.h new file mode 100644 index 0000000000..205c97ef95 --- /dev/null +++ b/src/3rd party/oneapi/tbb/parallel_reduce.h @@ -0,0 +1,772 @@ +/* + Copyright (c) 2005-2024 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_parallel_reduce_H +#define __TBB_parallel_reduce_H + +#include +#include "detail/_namespace_injection.h" +#include "detail/_task.h" +#include "detail/_aligned_space.h" +#include "detail/_small_object_pool.h" +#include "detail/_range_common.h" + +#include "task_group.h" // task_group_context +#include "partitioner.h" +#include "profiling.h" + +namespace tbb { +namespace detail { +#if __TBB_CPP20_CONCEPTS_PRESENT +inline namespace d0 { + +template +concept parallel_reduce_body = splittable && + requires( Body& body, const Range& range, Body& rhs ) { + body(range); + body.join(rhs); + }; + +template +concept parallel_reduce_function = std::invocable&, + const Range&, Value&&> && + std::convertible_to&, + const Range&, Value&&>, + Value>; + +template +concept parallel_reduce_combine = std::invocable&, + Value&&, Value&&> && + std::convertible_to&, + Value&&, Value&&>, + Value>; + +} // namespace d0 +#endif // __TBB_CPP20_CONCEPTS_PRESENT +namespace d1 { + +//! Tree node type for parallel_reduce. +/** @ingroup algorithms */ +//TODO: consider folding tree via bypass execution(instead of manual folding) +// for better cancellation and critical tasks handling (performance measurements required). +template +struct reduction_tree_node : public tree_node { + tbb::detail::aligned_space zombie_space; + Body& left_body; + bool has_right_zombie{false}; + + reduction_tree_node(node* parent, int ref_count, Body& input_left_body, small_object_allocator& alloc) : + tree_node{parent, ref_count, alloc}, + left_body(input_left_body) /* gcc4.8 bug - braced-initialization doesn't work for class members of reference type */ + {} + + void join(task_group_context* context) { + if (has_right_zombie && !context->is_group_execution_cancelled()) + left_body.join(*zombie_space.begin()); + } + + ~reduction_tree_node() { + if( has_right_zombie ) zombie_space.begin()->~Body(); + } +}; + +//! Task type used to split the work of parallel_reduce. +/** @ingroup algorithms */ +template +struct start_reduce : public task { + Range my_range; + Body* my_body; + node* my_parent; + + typename Partitioner::task_partition_type my_partition; + small_object_allocator my_allocator; + bool is_right_child; + + task* execute(execution_data&) override; + task* cancel(execution_data&) override; + void finalize(const execution_data&); + + using tree_node_type = reduction_tree_node; + + //! Constructor reduce root task. + start_reduce( const Range& range, Body& body, Partitioner& partitioner, small_object_allocator& alloc ) : + my_range(range), + my_body(&body), + my_parent(nullptr), + my_partition(partitioner), + my_allocator(alloc), + is_right_child(false) {} + //! Splitting constructor used to generate children. + /** parent_ becomes left child. Newly constructed object is right child. */ + start_reduce( start_reduce& parent_, typename Partitioner::split_type& split_obj, small_object_allocator& alloc ) : + my_range(parent_.my_range, get_range_split_object(split_obj)), + my_body(parent_.my_body), + my_parent(nullptr), + my_partition(parent_.my_partition, split_obj), + my_allocator(alloc), + is_right_child(true) + { + parent_.is_right_child = false; + } + //! Construct right child from the given range as response to the demand. + /** parent_ remains left child. Newly constructed object is right child. */ + start_reduce( start_reduce& parent_, const Range& r, depth_t d, small_object_allocator& alloc ) : + my_range(r), + my_body(parent_.my_body), + my_parent(nullptr), + my_partition(parent_.my_partition, split()), + my_allocator(alloc), + is_right_child(true) + { + my_partition.align_depth( d ); + parent_.is_right_child = false; + } + static void run(const Range& range, Body& body, Partitioner& partitioner, task_group_context& context) { + if ( !range.empty() ) { + wait_node wn; + small_object_allocator alloc{}; + auto reduce_task = alloc.new_object(range, body, partitioner, alloc); + reduce_task->my_parent = &wn; + execute_and_wait(*reduce_task, context, wn.m_wait, context); + } + } + static void run(const Range& range, Body& body, Partitioner& partitioner) { + // Bound context prevents exceptions from body to affect nesting or sibling algorithms, + // and allows users to handle exceptions safely by wrapping parallel_reduce in the try-block. + task_group_context context(PARALLEL_REDUCE); + run(range, body, partitioner, context); + } + //! Run body for range, serves as callback for partitioner + void run_body( Range &r ) { + tbb::detail::invoke(*my_body, r); + } + + //! spawn right task, serves as callback for partitioner + void offer_work(typename Partitioner::split_type& split_obj, execution_data& ed) { + offer_work_impl(ed, *this, split_obj); + } + //! spawn right task, serves as callback for partitioner + void offer_work(const Range& r, depth_t d, execution_data& ed) { + offer_work_impl(ed, *this, r, d); + } + +private: + template + void offer_work_impl(execution_data& ed, Args&&... args) { + small_object_allocator alloc{}; + // New right child + auto right_child = alloc.new_object(ed, std::forward(args)..., alloc); + + // New root node as a continuation and ref count. Left and right child attach to the new parent. + right_child->my_parent = my_parent = alloc.new_object(ed, my_parent, 2, *my_body, alloc); + + // Spawn the right sibling + right_child->spawn_self(ed); + } + + void spawn_self(execution_data& ed) { + my_partition.spawn_task(*this, *context(ed)); + } +}; + +//! fold the tree and deallocate the task +template +void start_reduce::finalize(const execution_data& ed) { + // Get the current parent and wait object before an object destruction + node* parent = my_parent; + auto allocator = my_allocator; + // Task execution finished - destroy it + this->~start_reduce(); + // Unwind the tree decrementing the parent`s reference count + fold_tree(parent, ed); + allocator.deallocate(this, ed); +} + +//! Execute parallel_reduce task +template +task* start_reduce::execute(execution_data& ed) { + if (!is_same_affinity(ed)) { + my_partition.note_affinity(execution_slot(ed)); + } + my_partition.check_being_stolen(*this, ed); + + // The acquire barrier synchronizes the data pointed with my_body if the left + // task has already finished. + __TBB_ASSERT(my_parent, nullptr); + if( is_right_child && my_parent->m_ref_count.load(std::memory_order_acquire) == 2 ) { + tree_node_type* parent_ptr = static_cast(my_parent); + my_body = static_cast(new( parent_ptr->zombie_space.begin() ) Body(*my_body, split())); + parent_ptr->has_right_zombie = true; + } + __TBB_ASSERT(my_body != nullptr, "Incorrect body value"); + + my_partition.execute(*this, my_range, ed); + + finalize(ed); + return nullptr; +} + +//! Cancel parallel_reduce task +template +task* start_reduce::cancel(execution_data& ed) { + finalize(ed); + return nullptr; +} + +//! Tree node type for parallel_deterministic_reduce. +/** @ingroup algorithms */ +template +struct deterministic_reduction_tree_node : public tree_node { + Body right_body; + Body& left_body; + + deterministic_reduction_tree_node(node* parent, int ref_count, Body& input_left_body, small_object_allocator& alloc) : + tree_node{parent, ref_count, alloc}, + right_body{input_left_body, detail::split()}, + left_body(input_left_body) + {} + + void join(task_group_context* context) { + if (!context->is_group_execution_cancelled()) + left_body.join(right_body); + } +}; + +//! Task type used to split the work of parallel_deterministic_reduce. +/** @ingroup algorithms */ +template +struct start_deterministic_reduce : public task { + Range my_range; + Body& my_body; + node* my_parent; + + typename Partitioner::task_partition_type my_partition; + small_object_allocator my_allocator; + + task* execute(execution_data&) override; + task* cancel(execution_data&) override; + void finalize(const execution_data&); + + using tree_node_type = deterministic_reduction_tree_node; + + //! Constructor deterministic_reduce root task. + start_deterministic_reduce( const Range& range, Partitioner& partitioner, Body& body, small_object_allocator& alloc ) : + my_range(range), + my_body(body), + my_parent(nullptr), + my_partition(partitioner), + my_allocator(alloc) {} + //! Splitting constructor used to generate children. + /** parent_ becomes left child. Newly constructed object is right child. */ + start_deterministic_reduce( start_deterministic_reduce& parent_, typename Partitioner::split_type& split_obj, Body& body, + small_object_allocator& alloc ) : + my_range(parent_.my_range, get_range_split_object(split_obj)), + my_body(body), + my_parent(nullptr), + my_partition(parent_.my_partition, split_obj), + my_allocator(alloc) {} + static void run(const Range& range, Body& body, Partitioner& partitioner, task_group_context& context) { + if ( !range.empty() ) { + wait_node wn; + small_object_allocator alloc{}; + auto deterministic_reduce_task = + alloc.new_object(range, partitioner, body, alloc); + deterministic_reduce_task->my_parent = &wn; + execute_and_wait(*deterministic_reduce_task, context, wn.m_wait, context); + } + } + static void run(const Range& range, Body& body, Partitioner& partitioner) { + // Bound context prevents exceptions from body to affect nesting or sibling algorithms, + // and allows users to handle exceptions safely by wrapping parallel_deterministic_reduce + // in the try-block. + task_group_context context(PARALLEL_REDUCE); + run(range, body, partitioner, context); + } + //! Run body for range, serves as callback for partitioner + void run_body( Range &r ) { + tbb::detail::invoke(my_body, r); + } + //! Spawn right task, serves as callback for partitioner + void offer_work(typename Partitioner::split_type& split_obj, execution_data& ed) { + offer_work_impl(ed, *this, split_obj); + } +private: + template + void offer_work_impl(execution_data& ed, Args&&... args) { + small_object_allocator alloc{}; + // New root node as a continuation and ref count. Left and right child attach to the new parent. Split the body. + auto new_tree_node = alloc.new_object(ed, my_parent, 2, my_body, alloc); + + // New right child + auto right_child = alloc.new_object(ed, std::forward(args)..., new_tree_node->right_body, alloc); + + right_child->my_parent = my_parent = new_tree_node; + + // Spawn the right sibling + right_child->spawn_self(ed); + } + + void spawn_self(execution_data& ed) { + my_partition.spawn_task(*this, *context(ed)); + } +}; + +//! Fold the tree and deallocate the task +template +void start_deterministic_reduce::finalize(const execution_data& ed) { + // Get the current parent and wait object before an object destruction + node* parent = my_parent; + + auto allocator = my_allocator; + // Task execution finished - destroy it + this->~start_deterministic_reduce(); + // Unwind the tree decrementing the parent`s reference count + fold_tree(parent, ed); + allocator.deallocate(this, ed); +} + +//! Execute parallel_deterministic_reduce task +template +task* start_deterministic_reduce::execute(execution_data& ed) { + if (!is_same_affinity(ed)) { + my_partition.note_affinity(execution_slot(ed)); + } + my_partition.check_being_stolen(*this, ed); + + my_partition.execute(*this, my_range, ed); + + finalize(ed); + return nullptr; +} + +//! Cancel parallel_deterministic_reduce task +template +task* start_deterministic_reduce::cancel(execution_data& ed) { + finalize(ed); + return nullptr; +} + + +//! Auxiliary class for parallel_reduce; for internal use only. +/** The adaptor class that implements \ref parallel_reduce_body_req "parallel_reduce Body" + using given \ref parallel_reduce_lambda_req "anonymous function objects". + **/ +/** @ingroup algorithms */ +template +class lambda_reduce_body { +//TODO: decide if my_real_body, my_reduction, and my_identity_element should be copied or referenced +// (might require some performance measurements) + + const Value& my_identity_element; + const RealBody& my_real_body; + const Reduction& my_reduction; + Value my_value; + lambda_reduce_body& operator= ( const lambda_reduce_body& other ); +public: + lambda_reduce_body( const Value& identity, const RealBody& body, const Reduction& reduction ) + : my_identity_element(identity) + , my_real_body(body) + , my_reduction(reduction) + , my_value(identity) + { } + lambda_reduce_body( const lambda_reduce_body& other ) = default; + lambda_reduce_body( lambda_reduce_body& other, tbb::split ) + : my_identity_element(other.my_identity_element) + , my_real_body(other.my_real_body) + , my_reduction(other.my_reduction) + , my_value(other.my_identity_element) + { } + void operator()(Range& range) { + my_value = tbb::detail::invoke(my_real_body, range, std::move(my_value)); + } + + void join( lambda_reduce_body& rhs ) { + my_value = tbb::detail::invoke(my_reduction, std::move(my_value), std::move(rhs.my_value)); + } + + __TBB_nodiscard Value&& result() && noexcept { + return std::move(my_value); + } +}; + + +// Requirements on Range concept are documented in blocked_range.h + +/** \page parallel_reduce_body_req Requirements on parallel_reduce body + Class \c Body implementing the concept of parallel_reduce body must define: + - \code Body::Body( Body&, split ); \endcode Splitting constructor. + Must be able to run concurrently with operator() and method \c join + - \code Body::~Body(); \endcode Destructor + - \code void Body::operator()( Range& r ); \endcode Function call operator applying body to range \c r + and accumulating the result + - \code void Body::join( Body& b ); \endcode Join results. + The result in \c b should be merged into the result of \c this +**/ + +/** \page parallel_reduce_lambda_req Requirements on parallel_reduce anonymous function objects (lambda functions) + TO BE DOCUMENTED +**/ + +/** \name parallel_reduce + See also requirements on \ref range_req "Range" and \ref parallel_reduce_body_req "parallel_reduce Body". **/ +//@{ + +//! Parallel iteration with reduction and default partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body ) { + start_reduce::run( range, body, __TBB_DEFAULT_PARTITIONER() ); +} + +//! Parallel iteration with reduction and simple_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, const simple_partitioner& partitioner ) { + start_reduce::run( range, body, partitioner ); +} + +//! Parallel iteration with reduction and auto_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, const auto_partitioner& partitioner ) { + start_reduce::run( range, body, partitioner ); +} + +//! Parallel iteration with reduction and static_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, const static_partitioner& partitioner ) { + start_reduce::run( range, body, partitioner ); +} + +//! Parallel iteration with reduction and affinity_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, affinity_partitioner& partitioner ) { + start_reduce::run( range, body, partitioner ); +} + +//! Parallel iteration with reduction, default partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, task_group_context& context ) { + start_reduce::run( range, body, __TBB_DEFAULT_PARTITIONER(), context ); +} + +//! Parallel iteration with reduction, simple partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, const simple_partitioner& partitioner, task_group_context& context ) { + start_reduce::run( range, body, partitioner, context ); +} + +//! Parallel iteration with reduction, auto_partitioner and user-supplied context +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, const auto_partitioner& partitioner, task_group_context& context ) { + start_reduce::run( range, body, partitioner, context ); +} + +//! Parallel iteration with reduction, static_partitioner and user-supplied context +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, const static_partitioner& partitioner, task_group_context& context ) { + start_reduce::run( range, body, partitioner, context ); +} + +//! Parallel iteration with reduction, affinity_partitioner and user-supplied context +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_reduce( const Range& range, Body& body, affinity_partitioner& partitioner, task_group_context& context ) { + start_reduce::run( range, body, partitioner, context ); +} +/** parallel_reduce overloads that work with anonymous function objects + (see also \ref parallel_reduce_lambda_req "requirements on parallel_reduce anonymous function objects"). **/ + +//! Parallel iteration with reduction and default partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,const __TBB_DEFAULT_PARTITIONER> + ::run(range, body, __TBB_DEFAULT_PARTITIONER() ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction and simple_partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + const simple_partitioner& partitioner ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,const simple_partitioner> + ::run(range, body, partitioner ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction and auto_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + const auto_partitioner& partitioner ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,const auto_partitioner> + ::run( range, body, partitioner ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction and static_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + const static_partitioner& partitioner ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,const static_partitioner> + ::run( range, body, partitioner ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction and affinity_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + affinity_partitioner& partitioner ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,affinity_partitioner> + ::run( range, body, partitioner ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction, default partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + task_group_context& context ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,const __TBB_DEFAULT_PARTITIONER> + ::run( range, body, __TBB_DEFAULT_PARTITIONER(), context ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction, simple partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + const simple_partitioner& partitioner, task_group_context& context ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,const simple_partitioner> + ::run( range, body, partitioner, context ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction, auto_partitioner and user-supplied context +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + const auto_partitioner& partitioner, task_group_context& context ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,const auto_partitioner> + ::run( range, body, partitioner, context ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction, static_partitioner and user-supplied context +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + const static_partitioner& partitioner, task_group_context& context ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,const static_partitioner> + ::run( range, body, partitioner, context ); + return std::move(body).result(); +} + +//! Parallel iteration with reduction, affinity_partitioner and user-supplied context +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + affinity_partitioner& partitioner, task_group_context& context ) { + lambda_reduce_body body(identity, real_body, reduction); + start_reduce,affinity_partitioner> + ::run( range, body, partitioner, context ); + return std::move(body).result(); +} + +//! Parallel iteration with deterministic reduction and default simple partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_deterministic_reduce( const Range& range, Body& body ) { + start_deterministic_reduce::run(range, body, simple_partitioner()); +} + +//! Parallel iteration with deterministic reduction and simple partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_deterministic_reduce( const Range& range, Body& body, const simple_partitioner& partitioner ) { + start_deterministic_reduce::run(range, body, partitioner); +} + +//! Parallel iteration with deterministic reduction and static partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_deterministic_reduce( const Range& range, Body& body, const static_partitioner& partitioner ) { + start_deterministic_reduce::run(range, body, partitioner); +} + +//! Parallel iteration with deterministic reduction, default simple partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_deterministic_reduce( const Range& range, Body& body, task_group_context& context ) { + start_deterministic_reduce::run( range, body, simple_partitioner(), context ); +} + +//! Parallel iteration with deterministic reduction, simple partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_deterministic_reduce( const Range& range, Body& body, const simple_partitioner& partitioner, task_group_context& context ) { + start_deterministic_reduce::run(range, body, partitioner, context); +} + +//! Parallel iteration with deterministic reduction, static partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_body) +void parallel_deterministic_reduce( const Range& range, Body& body, const static_partitioner& partitioner, task_group_context& context ) { + start_deterministic_reduce::run(range, body, partitioner, context); +} + +/** parallel_reduce overloads that work with anonymous function objects + (see also \ref parallel_reduce_lambda_req "requirements on parallel_reduce anonymous function objects"). **/ + +//! Parallel iteration with deterministic reduction and default simple partitioner. +// TODO: consider making static_partitioner the default +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction ) { + return parallel_deterministic_reduce(range, identity, real_body, reduction, simple_partitioner()); +} + +//! Parallel iteration with deterministic reduction and simple partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, const simple_partitioner& partitioner ) { + lambda_reduce_body body(identity, real_body, reduction); + start_deterministic_reduce, const simple_partitioner> + ::run(range, body, partitioner); + return std::move(body).result(); +} + +//! Parallel iteration with deterministic reduction and static partitioner. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, const static_partitioner& partitioner ) { + lambda_reduce_body body(identity, real_body, reduction); + start_deterministic_reduce, const static_partitioner> + ::run(range, body, partitioner); + return std::move(body).result(); +} + +//! Parallel iteration with deterministic reduction, default simple partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + task_group_context& context ) { + return parallel_deterministic_reduce(range, identity, real_body, reduction, simple_partitioner(), context); +} + +//! Parallel iteration with deterministic reduction, simple partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + const simple_partitioner& partitioner, task_group_context& context ) { + lambda_reduce_body body(identity, real_body, reduction); + start_deterministic_reduce, const simple_partitioner> + ::run(range, body, partitioner, context); + return std::move(body).result(); +} + +//! Parallel iteration with deterministic reduction, static partitioner and user-supplied context. +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_reduce_function && + parallel_reduce_combine) +Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, + const static_partitioner& partitioner, task_group_context& context ) { + lambda_reduce_body body(identity, real_body, reduction); + start_deterministic_reduce, const static_partitioner> + ::run(range, body, partitioner, context); + return std::move(body).result(); +} +//@} + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::parallel_reduce; +using detail::d1::parallel_deterministic_reduce; +// Split types +using detail::split; +using detail::proportional_split; +} // namespace v1 + +} // namespace tbb +#endif /* __TBB_parallel_reduce_H */ diff --git a/src/3rd party/oneapi/tbb/parallel_scan.h b/src/3rd party/oneapi/tbb/parallel_scan.h new file mode 100644 index 0000000000..8ef0777baa --- /dev/null +++ b/src/3rd party/oneapi/tbb/parallel_scan.h @@ -0,0 +1,630 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_parallel_scan_H +#define __TBB_parallel_scan_H + +#include + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_exception.h" +#include "detail/_task.h" + +#include "profiling.h" +#include "partitioner.h" +#include "blocked_range.h" +#include "task_group.h" + +namespace tbb { +namespace detail { +namespace d1 { + +//! Used to indicate that the initial scan is being performed. +/** @ingroup algorithms */ +struct pre_scan_tag { + static bool is_final_scan() {return false;} + operator bool() {return is_final_scan();} +}; + +//! Used to indicate that the final scan is being performed. +/** @ingroup algorithms */ +struct final_scan_tag { + static bool is_final_scan() {return true;} + operator bool() {return is_final_scan();} +}; + +template +struct sum_node; + +#if __TBB_CPP20_CONCEPTS_PRESENT +} // namespace d1 +inline namespace d0 { + +template +concept parallel_scan_body = splittable && + requires( Body& body, const Range& range, Body& other ) { + body(range, tbb::detail::d1::pre_scan_tag{}); + body(range, tbb::detail::d1::final_scan_tag{}); + body.reverse_join(other); + body.assign(other); + }; + +template +concept parallel_scan_function = std::invocable&, + const Range&, const Value&, bool> && + std::convertible_to&, + const Range&, const Value&, bool>, + Value>; + +template +concept parallel_scan_combine = std::invocable&, + const Value&, const Value&> && + std::convertible_to&, + const Value&, const Value&>, + Value>; + +} // namespace d0 +namespace d1 { +#endif // __TBB_CPP20_CONCEPTS_PRESENT + +//! Performs final scan for a leaf +/** @ingroup algorithms */ +template +struct final_sum : public task { +private: + using sum_node_type = sum_node; + Body m_body; + aligned_space m_range; + //! Where to put result of last subrange, or nullptr if not last subrange. + Body* m_stuff_last; + + wait_context& m_wait_context; + sum_node_type* m_parent = nullptr; +public: + small_object_allocator m_allocator; + final_sum( Body& body, wait_context& w_o, small_object_allocator& alloc ) : + m_body(body, split()), m_wait_context(w_o), m_allocator(alloc) { + poison_pointer(m_stuff_last); + } + + final_sum( final_sum& sum, small_object_allocator& alloc ) : + m_body(sum.m_body, split()), m_wait_context(sum.m_wait_context), m_allocator(alloc) { + poison_pointer(m_stuff_last); + } + + ~final_sum() { + m_range.begin()->~Range(); + } + void finish_construction( sum_node_type* parent, const Range& range, Body* stuff_last ) { + __TBB_ASSERT( m_parent == nullptr, nullptr ); + m_parent = parent; + new( m_range.begin() ) Range(range); + m_stuff_last = stuff_last; + } +private: + sum_node_type* release_parent() { + call_itt_task_notify(releasing, m_parent); + if (m_parent) { + auto parent = m_parent; + m_parent = nullptr; + if (parent->ref_count.fetch_sub(1) == 1) { + return parent; + } + } + else + m_wait_context.release(); + return nullptr; + } + sum_node_type* finalize(const execution_data& ed){ + sum_node_type* next_task = release_parent(); + m_allocator.delete_object(this, ed); + return next_task; + } + +public: + task* execute(execution_data& ed) override { + m_body( *m_range.begin(), final_scan_tag() ); + if( m_stuff_last ) + m_stuff_last->assign(m_body); + + return finalize(ed); + } + task* cancel(execution_data& ed) override { + return finalize(ed); + } + template + void operator()( const Range& r, Tag tag ) { + m_body( r, tag ); + } + void reverse_join( final_sum& a ) { + m_body.reverse_join(a.m_body); + } + void reverse_join( Body& body ) { + m_body.reverse_join(body); + } + void assign_to( Body& body ) { + body.assign(m_body); + } + void self_destroy(const execution_data& ed) { + m_allocator.delete_object(this, ed); + } +}; + +//! Split work to be done in the scan. +/** @ingroup algorithms */ +template +struct sum_node : public task { +private: + using final_sum_type = final_sum; +public: + final_sum_type *m_incoming; + final_sum_type *m_body; + Body *m_stuff_last; +private: + final_sum_type *m_left_sum; + sum_node *m_left; + sum_node *m_right; + bool m_left_is_final; + Range m_range; + wait_context& m_wait_context; + sum_node* m_parent; + small_object_allocator m_allocator; +public: + std::atomic ref_count{0}; + sum_node( const Range range, bool left_is_final_, sum_node* parent, wait_context& w_o, small_object_allocator& alloc ) : + m_stuff_last(nullptr), + m_left_sum(nullptr), + m_left(nullptr), + m_right(nullptr), + m_left_is_final(left_is_final_), + m_range(range), + m_wait_context(w_o), + m_parent(parent), + m_allocator(alloc) + { + if( m_parent ) + m_parent->ref_count.fetch_add(1); + // Poison fields that will be set by second pass. + poison_pointer(m_body); + poison_pointer(m_incoming); + } + + ~sum_node() { + if (m_parent) + m_parent->ref_count.fetch_sub(1); + } +private: + sum_node* release_parent() { + call_itt_task_notify(releasing, m_parent); + if (m_parent) { + auto parent = m_parent; + m_parent = nullptr; + if (parent->ref_count.fetch_sub(1) == 1) { + return parent; + } + } + else + m_wait_context.release(); + return nullptr; + } + task* create_child( const Range& range, final_sum_type& body, sum_node* child, final_sum_type* incoming, Body* stuff_last ) { + if( child ) { + __TBB_ASSERT( is_poisoned(child->m_body) && is_poisoned(child->m_incoming), nullptr ); + child->prepare_for_execution(body, incoming, stuff_last); + return child; + } else { + body.finish_construction(this, range, stuff_last); + return &body; + } + } + + sum_node* finalize(const execution_data& ed) { + sum_node* next_task = release_parent(); + m_allocator.delete_object(this, ed); + return next_task; + } + +public: + void prepare_for_execution(final_sum_type& body, final_sum_type* incoming, Body *stuff_last) { + this->m_body = &body; + this->m_incoming = incoming; + this->m_stuff_last = stuff_last; + } + task* execute(execution_data& ed) override { + if( m_body ) { + if( m_incoming ) + m_left_sum->reverse_join( *m_incoming ); + task* right_child = this->create_child(Range(m_range,split()), *m_left_sum, m_right, m_left_sum, m_stuff_last); + task* left_child = m_left_is_final ? nullptr : this->create_child(m_range, *m_body, m_left, m_incoming, nullptr); + ref_count = (left_child != nullptr) + (right_child != nullptr); + m_body = nullptr; + if( left_child ) { + spawn(*right_child, *ed.context); + return left_child; + } else { + return right_child; + } + } else { + return finalize(ed); + } + } + task* cancel(execution_data& ed) override { + return finalize(ed); + } + void self_destroy(const execution_data& ed) { + m_allocator.delete_object(this, ed); + } + template + friend struct start_scan; + + template + friend struct finish_scan; +}; + +//! Combine partial results +/** @ingroup algorithms */ +template +struct finish_scan : public task { +private: + using sum_node_type = sum_node; + using final_sum_type = final_sum; + final_sum_type** const m_sum_slot; + sum_node_type*& m_return_slot; + small_object_allocator m_allocator; +public: + std::atomic m_right_zombie; + sum_node_type& m_result; + std::atomic ref_count{2}; + finish_scan* m_parent; + wait_context& m_wait_context; + task* execute(execution_data& ed) override { + __TBB_ASSERT( m_result.ref_count.load() == static_cast((m_result.m_left!=nullptr)+(m_result.m_right!=nullptr)), nullptr ); + if( m_result.m_left ) + m_result.m_left_is_final = false; + final_sum_type* right_zombie = m_right_zombie.load(std::memory_order_acquire); + if( right_zombie && m_sum_slot ) + (*m_sum_slot)->reverse_join(*m_result.m_left_sum); + __TBB_ASSERT( !m_return_slot, nullptr ); + if( right_zombie || m_result.m_right ) { + m_return_slot = &m_result; + } else { + m_result.self_destroy(ed); + } + if( right_zombie && !m_sum_slot && !m_result.m_right ) { + right_zombie->self_destroy(ed); + m_right_zombie.store(nullptr, std::memory_order_relaxed); + } + return finalize(ed); + } + task* cancel(execution_data& ed) override { + return finalize(ed); + } + finish_scan(sum_node_type*& return_slot, final_sum_type** sum, sum_node_type& result_, finish_scan* parent, wait_context& w_o, small_object_allocator& alloc) : + m_sum_slot(sum), + m_return_slot(return_slot), + m_allocator(alloc), + m_right_zombie(nullptr), + m_result(result_), + m_parent(parent), + m_wait_context(w_o) + { + __TBB_ASSERT( !m_return_slot, nullptr ); + } +private: + finish_scan* release_parent() { + call_itt_task_notify(releasing, m_parent); + if (m_parent) { + auto parent = m_parent; + m_parent = nullptr; + if (parent->ref_count.fetch_sub(1) == 1) { + return parent; + } + } + else + m_wait_context.release(); + return nullptr; + } + finish_scan* finalize(const execution_data& ed) { + finish_scan* next_task = release_parent(); + m_allocator.delete_object(this, ed); + return next_task; + } +}; + +//! Initial task to split the work +/** @ingroup algorithms */ +template +struct start_scan : public task { +private: + using sum_node_type = sum_node; + using final_sum_type = final_sum; + using finish_pass1_type = finish_scan; + std::reference_wrapper m_return_slot; + Range m_range; + std::reference_wrapper m_body; + typename Partitioner::partition_type m_partition; + /** Non-null if caller is requesting total. */ + final_sum_type** m_sum_slot; + bool m_is_final; + bool m_is_right_child; + + finish_pass1_type* m_parent; + small_object_allocator m_allocator; + wait_context& m_wait_context; + + finish_pass1_type* release_parent() { + call_itt_task_notify(releasing, m_parent); + if (m_parent) { + auto parent = m_parent; + m_parent = nullptr; + if (parent->ref_count.fetch_sub(1) == 1) { + return parent; + } + } + else + m_wait_context.release(); + return nullptr; + } + + finish_pass1_type* finalize( const execution_data& ed ) { + finish_pass1_type* next_task = release_parent(); + m_allocator.delete_object(this, ed); + return next_task; + } + +public: + task* execute( execution_data& ) override; + task* cancel( execution_data& ed ) override { + return finalize(ed); + } + start_scan( sum_node_type*& return_slot, start_scan& parent, small_object_allocator& alloc ) : + m_return_slot(return_slot), + m_range(parent.m_range,split()), + m_body(parent.m_body), + m_partition(parent.m_partition,split()), + m_sum_slot(parent.m_sum_slot), + m_is_final(parent.m_is_final), + m_is_right_child(true), + m_parent(parent.m_parent), + m_allocator(alloc), + m_wait_context(parent.m_wait_context) + { + __TBB_ASSERT( !m_return_slot, nullptr ); + parent.m_is_right_child = false; + } + + start_scan( sum_node_type*& return_slot, const Range& range, final_sum_type& body, const Partitioner& partitioner, wait_context& w_o, small_object_allocator& alloc ) : + m_return_slot(return_slot), + m_range(range), + m_body(body), + m_partition(partitioner), + m_sum_slot(nullptr), + m_is_final(true), + m_is_right_child(false), + m_parent(nullptr), + m_allocator(alloc), + m_wait_context(w_o) + { + __TBB_ASSERT( !m_return_slot, nullptr ); + } + + static void run( const Range& range, Body& body, const Partitioner& partitioner ) { + if( !range.empty() ) { + task_group_context context(PARALLEL_SCAN); + + using start_pass1_type = start_scan; + sum_node_type* root = nullptr; + wait_context w_ctx{1}; + small_object_allocator alloc{}; + + auto& temp_body = *alloc.new_object(body, w_ctx, alloc); + temp_body.reverse_join(body); + + auto& pass1 = *alloc.new_object(/*m_return_slot=*/root, range, temp_body, partitioner, w_ctx, alloc); + + execute_and_wait(pass1, context, w_ctx, context); + if( root ) { + root->prepare_for_execution(temp_body, nullptr, &body); + w_ctx.reserve(); + execute_and_wait(*root, context, w_ctx, context); + } else { + temp_body.assign_to(body); + temp_body.finish_construction(nullptr, range, nullptr); + alloc.delete_object(&temp_body); + } + } + } +}; + +template +task* start_scan::execute( execution_data& ed ) { + // Inspecting m_parent->result.left_sum would ordinarily be a race condition. + // But we inspect it only if we are not a stolen task, in which case we + // know that task assigning to m_parent->result.left_sum has completed. + __TBB_ASSERT(!m_is_right_child || m_parent, "right child is never an orphan"); + bool treat_as_stolen = m_is_right_child && (is_stolen(ed) || &m_body.get()!=m_parent->m_result.m_left_sum); + if( treat_as_stolen ) { + // Invocation is for right child that has been really stolen or needs to be virtually stolen + small_object_allocator alloc{}; + final_sum_type* right_zombie = alloc.new_object(m_body, alloc); + m_parent->m_right_zombie.store(right_zombie, std::memory_order_release); + m_body = *right_zombie; + m_is_final = false; + } + task* next_task = nullptr; + if( (m_is_right_child && !treat_as_stolen) || !m_range.is_divisible() || m_partition.should_execute_range(ed) ) { + if( m_is_final ) + m_body(m_range, final_scan_tag()); + else if( m_sum_slot ) + m_body(m_range, pre_scan_tag()); + if( m_sum_slot ) + *m_sum_slot = &m_body.get(); + __TBB_ASSERT( !m_return_slot, nullptr ); + + next_task = finalize(ed); + } else { + small_object_allocator alloc{}; + auto result = alloc.new_object(m_range,/*m_left_is_final=*/m_is_final, m_parent? &m_parent->m_result: nullptr, m_wait_context, alloc); + + auto new_parent = alloc.new_object(m_return_slot, m_sum_slot, *result, m_parent, m_wait_context, alloc); + m_parent = new_parent; + + // Split off right child + auto& right_child = *alloc.new_object(/*m_return_slot=*/result->m_right, *this, alloc); + + spawn(right_child, *ed.context); + + m_sum_slot = &result->m_left_sum; + m_return_slot = result->m_left; + + __TBB_ASSERT( !m_return_slot, nullptr ); + next_task = this; + } + return next_task; +} + +template +class lambda_scan_body { + Value m_sum_slot; + const Value& identity_element; + const Scan& m_scan; + const ReverseJoin& m_reverse_join; +public: + void operator=(const lambda_scan_body&) = delete; + lambda_scan_body(const lambda_scan_body&) = default; + + lambda_scan_body( const Value& identity, const Scan& scan, const ReverseJoin& rev_join ) + : m_sum_slot(identity) + , identity_element(identity) + , m_scan(scan) + , m_reverse_join(rev_join) {} + + lambda_scan_body( lambda_scan_body& b, split ) + : m_sum_slot(b.identity_element) + , identity_element(b.identity_element) + , m_scan(b.m_scan) + , m_reverse_join(b.m_reverse_join) {} + + template + void operator()( const Range& r, Tag tag ) { + m_sum_slot = tbb::detail::invoke(m_scan, r, m_sum_slot, tag); + } + + void reverse_join( lambda_scan_body& a ) { + m_sum_slot = tbb::detail::invoke(m_reverse_join, a.m_sum_slot, m_sum_slot); + } + + void assign( lambda_scan_body& b ) { + m_sum_slot = b.m_sum_slot; + } + + Value result() const { + return m_sum_slot; + } +}; + +// Requirements on Range concept are documented in blocked_range.h + +/** \page parallel_scan_body_req Requirements on parallel_scan body + Class \c Body implementing the concept of parallel_scan body must define: + - \code Body::Body( Body&, split ); \endcode Splitting constructor. + Split \c b so that \c this and \c b can accumulate separately + - \code Body::~Body(); \endcode Destructor + - \code void Body::operator()( const Range& r, pre_scan_tag ); \endcode + Preprocess iterations for range \c r + - \code void Body::operator()( const Range& r, final_scan_tag ); \endcode + Do final processing for iterations of range \c r + - \code void Body::reverse_join( Body& a ); \endcode + Merge preprocessing state of \c a into \c this, where \c a was + created earlier from \c b by b's splitting constructor +**/ + +/** \name parallel_scan + See also requirements on \ref range_req "Range" and \ref parallel_scan_body_req "parallel_scan Body". **/ +//@{ + +//! Parallel prefix with default partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_scan_body) +void parallel_scan( const Range& range, Body& body ) { + start_scan::run(range,body,__TBB_DEFAULT_PARTITIONER()); +} + +//! Parallel prefix with simple_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_scan_body) +void parallel_scan( const Range& range, Body& body, const simple_partitioner& partitioner ) { + start_scan::run(range, body, partitioner); +} + +//! Parallel prefix with auto_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_scan_body) +void parallel_scan( const Range& range, Body& body, const auto_partitioner& partitioner ) { + start_scan::run(range, body, partitioner); +} + +//! Parallel prefix with default partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_scan_function && + parallel_scan_combine) +Value parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join ) { + lambda_scan_body body(identity, scan, reverse_join); + parallel_scan(range, body, __TBB_DEFAULT_PARTITIONER()); + return body.result(); +} + +//! Parallel prefix with simple_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_scan_function && + parallel_scan_combine) +Value parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join, + const simple_partitioner& partitioner ) { + lambda_scan_body body(identity, scan, reverse_join); + parallel_scan(range, body, partitioner); + return body.result(); +} + +//! Parallel prefix with auto_partitioner +/** @ingroup algorithms **/ +template + __TBB_requires(tbb_range && parallel_scan_function && + parallel_scan_combine) +Value parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join, + const auto_partitioner& partitioner ) { + lambda_scan_body body(identity, scan, reverse_join); + parallel_scan(range, body, partitioner); + return body.result(); +} + +} // namespace d1 +} // namespace detail + +inline namespace v1 { + using detail::d1::parallel_scan; + using detail::d1::pre_scan_tag; + using detail::d1::final_scan_tag; +} // namespace v1 + +} // namespace tbb + +#endif /* __TBB_parallel_scan_H */ diff --git a/src/3rd party/oneapi/tbb/parallel_sort.h b/src/3rd party/oneapi/tbb/parallel_sort.h new file mode 100644 index 0000000000..4c0578eb09 --- /dev/null +++ b/src/3rd party/oneapi/tbb/parallel_sort.h @@ -0,0 +1,288 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_parallel_sort_H +#define __TBB_parallel_sort_H + +#include "detail/_namespace_injection.h" +#include "parallel_for.h" +#include "blocked_range.h" +#include "profiling.h" + +#include +#include +#include +#include + +namespace tbb { +namespace detail { +#if __TBB_CPP20_CONCEPTS_PRESENT +inline namespace d0 { + +// TODO: consider using std::strict_weak_order concept +template +concept compare = requires( const std::remove_reference_t& comp, typename std::iterator_traits::reference value ) { + // Forward via iterator_traits::reference + { comp(typename std::iterator_traits::reference(value), + typename std::iterator_traits::reference(value)) } -> std::convertible_to; +}; + +// Inspired by std::__PartiallyOrderedWith exposition only concept +template +concept less_than_comparable = requires( const std::remove_reference_t& lhs, + const std::remove_reference_t& rhs ) { + { lhs < rhs } -> boolean_testable; +}; + +} // namespace d0 +#endif // __TBB_CPP20_CONCEPTS_PRESENT +namespace d1 { + +//! Range used in quicksort to split elements into subranges based on a value. +/** The split operation selects a splitter and places all elements less than or equal + to the value in the first range and the remaining elements in the second range. + @ingroup algorithms */ +template +class quick_sort_range { + std::size_t median_of_three( const RandomAccessIterator& array, std::size_t l, std::size_t m, std::size_t r ) const { + return comp(array[l], array[m]) ? ( comp(array[m], array[r]) ? m : ( comp(array[l], array[r]) ? r : l ) ) + : ( comp(array[r], array[m]) ? m : ( comp(array[r], array[l]) ? r : l ) ); + } + + std::size_t pseudo_median_of_nine( const RandomAccessIterator& array, const quick_sort_range& range ) const { + std::size_t offset = range.size / 8u; + return median_of_three(array, + median_of_three(array, 0 , offset, offset * 2), + median_of_three(array, offset * 3, offset * 4, offset * 5), + median_of_three(array, offset * 6, offset * 7, range.size - 1)); + + } + + std::size_t split_range( quick_sort_range& range ) { + RandomAccessIterator array = range.begin; + RandomAccessIterator first_element = range.begin; + std::size_t m = pseudo_median_of_nine(array, range); + if( m != 0 ) std::iter_swap(array, array + m); + + std::size_t i = 0; + std::size_t j = range.size; + // Partition interval [i + 1,j - 1] with key *first_element. + for(;;) { + __TBB_ASSERT( i < j, nullptr ); + // Loop must terminate since array[l] == *first_element. + do { + --j; + __TBB_ASSERT( i <= j, "bad ordering relation?" ); + } while( comp(*first_element, array[j]) ); + do { + __TBB_ASSERT( i <= j, nullptr ); + if( i == j ) goto partition; + ++i; + } while( comp(array[i], *first_element) ); + if( i == j ) goto partition; + std::iter_swap(array + i, array + j); + } +partition: + // Put the partition key were it belongs + std::iter_swap(array + j, first_element); + // array[l..j) is less or equal to key. + // array(j..r) is greater or equal to key. + // array[j] is equal to key + i = j + 1; + std::size_t new_range_size = range.size - i; + range.size = j; + return new_range_size; + } + +public: + quick_sort_range() = default; + quick_sort_range( const quick_sort_range& ) = default; + void operator=( const quick_sort_range& ) = delete; + + static constexpr std::size_t grainsize = 500; + const Compare& comp; + std::size_t size; + RandomAccessIterator begin; + + quick_sort_range( RandomAccessIterator begin_, std::size_t size_, const Compare& comp_ ) : + comp(comp_), size(size_), begin(begin_) {} + + bool empty() const { return size == 0; } + bool is_divisible() const { return size >= grainsize; } + + quick_sort_range( quick_sort_range& range, split ) + : comp(range.comp) + , size(split_range(range)) + // +1 accounts for the pivot element, which is at its correct place + // already and, therefore, is not included into subranges. + , begin(range.begin + range.size + 1) {} +}; + +//! Body class used to test if elements in a range are presorted +/** @ingroup algorithms */ +template +class quick_sort_pretest_body { + const Compare& comp; + task_group_context& context; + +public: + quick_sort_pretest_body() = default; + quick_sort_pretest_body( const quick_sort_pretest_body& ) = default; + void operator=( const quick_sort_pretest_body& ) = delete; + + quick_sort_pretest_body( const Compare& _comp, task_group_context& _context ) : comp(_comp), context(_context) {} + + void operator()( const blocked_range& range ) const { + RandomAccessIterator my_end = range.end(); + + int i = 0; + //TODO: consider using std::is_sorted() for each 64 iterations (requires performance measurements) + for( RandomAccessIterator k = range.begin(); k != my_end; ++k, ++i ) { + if( i % 64 == 0 && context.is_group_execution_cancelled() ) break; + + // The k - 1 is never out-of-range because the first chunk starts at begin+serial_cutoff+1 + if( comp(*(k), *(k - 1)) ) { + context.cancel_group_execution(); + break; + } + } + } +}; + +//! Body class used to sort elements in a range that is smaller than the grainsize. +/** @ingroup algorithms */ +template +struct quick_sort_body { + void operator()( const quick_sort_range& range ) const { + std::sort(range.begin, range.begin + range.size, range.comp); + } +}; + +//! Method to perform parallel_for based quick sort. +/** @ingroup algorithms */ +template +void do_parallel_quick_sort( RandomAccessIterator begin, RandomAccessIterator end, const Compare& comp ) { + parallel_for(quick_sort_range(begin, end - begin, comp), + quick_sort_body(), + auto_partitioner()); +} + +//! Wrapper method to initiate the sort by calling parallel_for. +/** @ingroup algorithms */ +template +void parallel_quick_sort( RandomAccessIterator begin, RandomAccessIterator end, const Compare& comp ) { + task_group_context my_context(PARALLEL_SORT); + constexpr int serial_cutoff = 9; + + __TBB_ASSERT( begin + serial_cutoff < end, "min_parallel_size is smaller than serial cutoff?" ); + RandomAccessIterator k = begin; + for( ; k != begin + serial_cutoff; ++k ) { + if( comp(*(k + 1), *k) ) { + do_parallel_quick_sort(begin, end, comp); + return; + } + } + + // Check is input range already sorted + parallel_for(blocked_range(k + 1, end), + quick_sort_pretest_body(comp, my_context), + auto_partitioner(), + my_context); + + if( my_context.is_group_execution_cancelled() ) + do_parallel_quick_sort(begin, end, comp); +} + +/** \page parallel_sort_iter_req Requirements on iterators for parallel_sort + Requirements on the iterator type \c It and its value type \c T for \c parallel_sort: + + - \code void iter_swap( It a, It b ) \endcode Swaps the values of the elements the given + iterators \c a and \c b are pointing to. \c It should be a random access iterator. + + - \code bool Compare::operator()( const T& x, const T& y ) \endcode True if x comes before y; +**/ + +/** \name parallel_sort + See also requirements on \ref parallel_sort_iter_req "iterators for parallel_sort". **/ +//@{ + +#if __TBB_CPP20_CONCEPTS_PRESENT +template +using iter_value_type = typename std::iterator_traits::value_type; + +template +using range_value_type = typename std::iterator_traits>::value_type; +#endif + +//! Sorts the data in [begin,end) using the given comparator +/** The compare function object is used for all comparisons between elements during sorting. + The compare object must define a bool operator() function. + @ingroup algorithms **/ +template + __TBB_requires(std::random_access_iterator && + compare && + std::movable>) +void parallel_sort( RandomAccessIterator begin, RandomAccessIterator end, const Compare& comp ) { + constexpr int min_parallel_size = 500; + if( end > begin ) { + if( end - begin < min_parallel_size ) { + std::sort(begin, end, comp); + } else { + parallel_quick_sort(begin, end, comp); + } + } +} + +//! Sorts the data in [begin,end) with a default comparator \c std::less +/** @ingroup algorithms **/ +template + __TBB_requires(std::random_access_iterator && + less_than_comparable> && + std::movable>) +void parallel_sort( RandomAccessIterator begin, RandomAccessIterator end ) { + parallel_sort(begin, end, std::less::value_type>()); +} + +//! Sorts the data in rng using the given comparator +/** @ingroup algorithms **/ +template + __TBB_requires(container_based_sequence && + compare> && + std::movable>) +void parallel_sort( Range&& rng, const Compare& comp ) { + parallel_sort(std::begin(rng), std::end(rng), comp); +} + +//! Sorts the data in rng with a default comparator \c std::less +/** @ingroup algorithms **/ +template + __TBB_requires(container_based_sequence && + less_than_comparable> && + std::movable>) +void parallel_sort( Range&& rng ) { + parallel_sort(std::begin(rng), std::end(rng)); +} +//@} + +} // namespace d1 +} // namespace detail + +inline namespace v1 { + using detail::d1::parallel_sort; +} // namespace v1 +} // namespace tbb + +#endif /*__TBB_parallel_sort_H*/ diff --git a/src/3rd party/oneapi/tbb/partitioner.h b/src/3rd party/oneapi/tbb/partitioner.h new file mode 100644 index 0000000000..870511e31a --- /dev/null +++ b/src/3rd party/oneapi/tbb/partitioner.h @@ -0,0 +1,682 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + Copyright (c) 2026 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_partitioner_H +#define __TBB_partitioner_H + +#ifndef __TBB_INITIAL_CHUNKS +// initial task divisions per thread +#define __TBB_INITIAL_CHUNKS 2 +#endif +#ifndef __TBB_RANGE_POOL_CAPACITY +// maximum number of elements in range pool +#define __TBB_RANGE_POOL_CAPACITY 8 +#endif +#ifndef __TBB_INIT_DEPTH +// initial value for depth of range pool +#define __TBB_INIT_DEPTH 5 +#endif +#ifndef __TBB_DEMAND_DEPTH_ADD +// when imbalance is found range splits this value times more +#define __TBB_DEMAND_DEPTH_ADD 1 +#endif + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_aligned_space.h" +#include "detail/_utils.h" +#include "detail/_template_helpers.h" +#include "detail/_range_common.h" +#include "detail/_task.h" +#include "detail/_small_object_pool.h" + +#include "cache_aligned_allocator.h" +#include "task_group.h" // task_group_context +#include "task_arena.h" + +#include +#include +#include + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + // Workaround for overzealous compiler warnings + #pragma warning (push) + #pragma warning (disable: 4244) +#endif + +namespace tbb { +namespace detail { + +namespace d1 { +class auto_partitioner; +class simple_partitioner; +class static_partitioner; +class affinity_partitioner; +class affinity_partition_type; +class affinity_partitioner_base; + +inline std::size_t get_initial_auto_partitioner_divisor() { + const std::size_t factor = 4; + return factor * static_cast(max_concurrency()); +} + +//! Defines entry point for affinity partitioner into oneTBB run-time library. +class affinity_partitioner_base: no_copy { + friend class affinity_partitioner; + friend class affinity_partition_type; + //! Array that remembers affinities of tree positions to affinity_id. + /** nullptr if my_size==0. */ + slot_id* my_array; + //! Number of elements in my_array. + std::size_t my_size; + //! Zeros the fields. + affinity_partitioner_base() : my_array(nullptr), my_size(0) {} + //! Deallocates my_array. + ~affinity_partitioner_base() { resize(0); } + //! Resize my_array. + /** Retains values if resulting size is the same. */ + void resize(unsigned factor) { + // Check factor to avoid asking for number of workers while there might be no arena. + unsigned max_threads_in_arena = static_cast(max_concurrency()); + std::size_t new_size = factor ? factor * max_threads_in_arena : 0; + if (new_size != my_size) { + if (my_array) { + r1::cache_aligned_deallocate(my_array); + // Following two assignments must be done here for sake of exception safety. + my_array = nullptr; + my_size = 0; + } + if (new_size) { + my_array = static_cast(r1::cache_aligned_allocate(new_size * sizeof(slot_id))); + std::fill_n(my_array, new_size, no_slot); + my_size = new_size; + } + } + } +}; + +template struct start_for; +template struct start_scan; +template struct start_reduce; +template struct start_deterministic_reduce; + +struct node { + node* my_parent{}; + std::atomic m_ref_count{}; + + node() = default; + node(node* parent, int ref_count) : + my_parent{parent}, m_ref_count{ref_count} { + __TBB_ASSERT(ref_count > 0, "The ref count must be positive"); + } +}; + +struct wait_node : node { + wait_node() : node{ nullptr, 1 } {} + wait_context m_wait{1}; +}; + +//! Join task node that contains shared flag for stealing feedback +struct tree_node : public node { + small_object_allocator m_allocator; + std::atomic m_child_stolen{false}; + + tree_node(node* parent, int ref_count, small_object_allocator& alloc) + : node{parent, ref_count} + , m_allocator{alloc} {} + + void join(task_group_context*) {/*dummy, required only for reduction algorithms*/}; + + template + static void mark_task_stolen(Task &t) { + std::atomic &flag = static_cast(t.my_parent)->m_child_stolen; +#if TBB_USE_PROFILING_TOOLS + // Threading tools respect lock prefix but report false-positive data-race via plain store + flag.exchange(true); +#else + flag.store(true, std::memory_order_relaxed); +#endif // TBB_USE_PROFILING_TOOLS + } + template + static bool is_peer_stolen(Task &t) { + return static_cast(t.my_parent)->m_child_stolen.load(std::memory_order_relaxed); + } +}; + +// Context used to check cancellation state during reduction join process +template +void fold_tree(node* n, const execution_data& ed) { + for (;;) { + __TBB_ASSERT(n, nullptr); + __TBB_ASSERT(n->m_ref_count.load(std::memory_order_relaxed) > 0, "The refcount must be positive."); + call_itt_task_notify(releasing, n); + if (--n->m_ref_count > 0) { + return; + } + node* parent = n->my_parent; + if (!parent) { + break; + }; + + call_itt_task_notify(acquired, n); + TreeNodeType* self = static_cast(n); + self->join(ed.context); + self->m_allocator.delete_object(self, ed); + n = parent; + } + // Finish parallel for execution when the root (last node) is reached + static_cast(n)->m_wait.release(); +} + +//! Depth is a relative depth of recursive division inside a range pool. Relative depth allows +//! infinite absolute depth of the recursion for heavily unbalanced workloads with range represented +//! by a number that cannot fit into machine word. +typedef unsigned char depth_t; + +//! Range pool stores ranges of type T in a circular buffer with MaxCapacity +template +class range_vector { + depth_t my_head; + depth_t my_tail; + depth_t my_size; + depth_t my_depth[MaxCapacity]; // relative depths of stored ranges + tbb::detail::aligned_space my_pool; + +public: + //! initialize via first range in pool + range_vector(const T& elem) : my_head(0), my_tail(0), my_size(1) { + my_depth[0] = 0; + new( static_cast(my_pool.begin()) ) T(elem);//TODO: std::move? + } + ~range_vector() { + while( !empty() ) pop_back(); + } + bool empty() const { return my_size == 0; } + depth_t size() const { return my_size; } + //! Populates range pool via ranges up to max depth or while divisible + //! max_depth starts from 0, e.g. value 2 makes 3 ranges in the pool up to two 1/4 pieces + void split_to_fill(depth_t max_depth) { + while( my_size < MaxCapacity && is_divisible(max_depth) ) { + depth_t prev = my_head; + my_head = (my_head + 1) % MaxCapacity; + new(my_pool.begin()+my_head) T(my_pool.begin()[prev]); // copy TODO: std::move? + my_pool.begin()[prev].~T(); // instead of assignment + new(my_pool.begin()+prev) T(my_pool.begin()[my_head], detail::split()); // do 'inverse' split + my_depth[my_head] = ++my_depth[prev]; + my_size++; + } + } + void pop_back() { + __TBB_ASSERT(my_size > 0, "range_vector::pop_back() with empty size"); + my_pool.begin()[my_head].~T(); + my_size--; + my_head = (my_head + MaxCapacity - 1) % MaxCapacity; + } + void pop_front() { + __TBB_ASSERT(my_size > 0, "range_vector::pop_front() with empty size"); + my_pool.begin()[my_tail].~T(); + my_size--; + my_tail = (my_tail + 1) % MaxCapacity; + } + T& back() { + __TBB_ASSERT(my_size > 0, "range_vector::back() with empty size"); + return my_pool.begin()[my_head]; + } + T& front() { + __TBB_ASSERT(my_size > 0, "range_vector::front() with empty size"); + return my_pool.begin()[my_tail]; + } + //! similarly to front(), returns depth of the first range in the pool + depth_t front_depth() { + __TBB_ASSERT(my_size > 0, "range_vector::front_depth() with empty size"); + return my_depth[my_tail]; + } + depth_t back_depth() { + __TBB_ASSERT(my_size > 0, "range_vector::back_depth() with empty size"); + return my_depth[my_head]; + } + bool is_divisible(depth_t max_depth) { + return back_depth() < max_depth && back().is_divisible(); + } +}; + +//! Provides default methods for partition objects and common algorithm blocks. +template +struct partition_type_base { + typedef detail::split split_type; + // decision makers + void note_affinity( slot_id ) {} + template + bool check_being_stolen(Task&, const execution_data&) { return false; } // part of old should_execute_range() + template split_type get_split() { return split(); } + Partition& self() { return *static_cast(this); } // CRTP helper + + template + void work_balance(StartType &start, Range &range, const execution_data&) { + start.run_body( range ); // static partitioner goes here + } + + template + void execute(StartType &start, Range &range, execution_data& ed) { + // The algorithm in a few words ([]-denotes calls to decision methods of partitioner): + // [If this task is stolen, adjust depth and divisions if necessary, set flag]. + // If range is divisible { + // Spread the work while [initial divisions left]; + // Create trap task [if necessary]; + // } + // If not divisible or [max depth is reached], execute, else do the range pool part + if ( range.is_divisible() ) { + if ( self().is_divisible() ) { + do { // split until is divisible + typename Partition::split_type split_obj = self().template get_split(); + start.offer_work( split_obj, ed ); + } while ( range.is_divisible() && self().is_divisible() ); + } + } + self().work_balance(start, range, ed); + } +}; + +//! Provides default splitting strategy for partition objects. +template +struct adaptive_mode : partition_type_base { + typedef Partition my_partition; + std::size_t my_divisor; + // For affinity_partitioner, my_divisor indicates the number of affinity array indices the task reserves. + // A task which has only one index must produce the right split without reserved index in order to avoid + // it to be overwritten in note_affinity() of the created (right) task. + // I.e. a task created deeper than the affinity array can remember must not save its affinity (LIFO order) + static const unsigned factor = 1; + adaptive_mode() : my_divisor(get_initial_auto_partitioner_divisor() / 4 * my_partition::factor) {} + adaptive_mode(adaptive_mode &src, split) : my_divisor(do_split(src, split())) {} + adaptive_mode(adaptive_mode&, const proportional_split&) : my_divisor(0) + { + // left blank as my_divisor gets overridden in the successors' constructors + } + /*! Override do_split methods in order to specify splitting strategy */ + std::size_t do_split(adaptive_mode &src, split) { + return src.my_divisor /= 2u; + } +}; + + +//! Provides proportional splitting strategy for partition objects +template +struct proportional_mode : adaptive_mode { + typedef Partition my_partition; + using partition_type_base::self; // CRTP helper to get access to derived classes + + proportional_mode() : adaptive_mode() {} + proportional_mode(proportional_mode &src, split) : adaptive_mode(src, split()) {} + proportional_mode(proportional_mode &src, const proportional_split& split_obj) + : adaptive_mode(src, split_obj) + { + self().my_divisor = do_split(src, split_obj); + } + std::size_t do_split(proportional_mode &src, const proportional_split& split_obj) { + std::size_t portion = split_obj.right() * my_partition::factor; + portion = (portion + my_partition::factor/2) & (0ul - my_partition::factor); + src.my_divisor -= portion; + return portion; + } + bool is_divisible() { // part of old should_execute_range() + return self().my_divisor > my_partition::factor; + } + template + proportional_split get_split() { + // Create the proportion from partitioner internal resources (threads) that would be used: + // - into proportional_mode constructor to split the partitioner + // - if Range supports the proportional_split constructor it would use proposed proportion, + // otherwise, the tbb::proportional_split object will be implicitly (for Range implementer) + // casted to tbb::split + + std::size_t n = self().my_divisor / my_partition::factor; + std::size_t right = n / 2; + std::size_t left = n - right; + return proportional_split(left, right); + } +}; + +inline std::size_t get_initial_partition_head() { + int current_index = tbb::this_task_arena::current_thread_index(); + if (current_index == tbb::task_arena::not_initialized) + current_index = 0; + return size_t(current_index); +} + +//! Provides default linear indexing of partitioner's sequence +template +struct linear_affinity_mode : proportional_mode { + std::size_t my_head; + std::size_t my_max_affinity; + using proportional_mode::self; + linear_affinity_mode() : proportional_mode(), my_head(get_initial_partition_head()), + my_max_affinity(self().my_divisor) {} + linear_affinity_mode(linear_affinity_mode &src, split) : proportional_mode(src, split()) + , my_head((src.my_head + src.my_divisor) % src.my_max_affinity), my_max_affinity(src.my_max_affinity) {} + linear_affinity_mode(linear_affinity_mode &src, const proportional_split& split_obj) : proportional_mode(src, split_obj) + , my_head((src.my_head + src.my_divisor) % src.my_max_affinity), my_max_affinity(src.my_max_affinity) {} + void spawn_task(task& t, task_group_context& ctx) { + if (self().my_divisor) { + spawn(t, ctx, slot_id(my_head)); + } else { + spawn(t, ctx); + } + } +}; + +inline bool is_stolen_task(const execution_data& ed) { + return execution_slot(ed) != original_slot(ed); +} + +/*! Determine work-balance phase implementing splitting & stealing actions */ +template +struct dynamic_grainsize_mode : Mode { + using Mode::self; + enum { + begin = 0, + run, + pass + } my_delay; + depth_t my_max_depth; + static const unsigned range_pool_size = __TBB_RANGE_POOL_CAPACITY; + dynamic_grainsize_mode(): Mode() + , my_delay(begin) + , my_max_depth(__TBB_INIT_DEPTH) {} + dynamic_grainsize_mode(dynamic_grainsize_mode& p, split) + : Mode(p, split()) + , my_delay(pass) + , my_max_depth(p.my_max_depth) {} + dynamic_grainsize_mode(dynamic_grainsize_mode& p, const proportional_split& split_obj) + : Mode(p, split_obj) + , my_delay(begin) + , my_max_depth(p.my_max_depth) {} + template + bool check_being_stolen(Task &t, const execution_data& ed) { // part of old should_execute_range() + if( !(self().my_divisor / Mode::my_partition::factor) ) { // if not from the top P tasks of binary tree + self().my_divisor = 1; // TODO: replace by on-stack flag (partition_state's member)? + if( is_stolen_task(ed) && t.my_parent->m_ref_count >= 2 ) { // runs concurrently with the left task +#if __TBB_USE_OPTIONAL_RTTI + // RTTI is available, check whether the cast is valid + // TODO: TBB_REVAMP_TODO __TBB_ASSERT(dynamic_cast(t.m_parent), 0); + // correctness of the cast relies on avoiding the root task for which: + // - initial value of my_divisor != 0 (protected by separate assertion) + // - is_stolen_task() always returns false for the root task. +#endif + tree_node::mark_task_stolen(t); + if( !my_max_depth ) my_max_depth++; + my_max_depth += __TBB_DEMAND_DEPTH_ADD; + return true; + } + } + return false; + } + depth_t max_depth() { return my_max_depth; } + void align_depth(depth_t base) { + __TBB_ASSERT(base <= my_max_depth, nullptr); + my_max_depth -= base; + } + template + void work_balance(StartType &start, Range &range, execution_data& ed) { + if( !range.is_divisible() || !self().max_depth() ) { + start.run_body( range ); + } + else { // do range pool + range_vector range_pool(range); + do { + range_pool.split_to_fill(self().max_depth()); // fill range pool + if( self().check_for_demand( start ) ) { + if( range_pool.size() > 1 ) { + start.offer_work( range_pool.front(), range_pool.front_depth(), ed ); + range_pool.pop_front(); + continue; + } + if( range_pool.is_divisible(self().max_depth()) ) // was not enough depth to fork a task + continue; // note: next split_to_fill() should split range at least once + } + start.run_body( range_pool.back() ); + range_pool.pop_back(); + } while( !range_pool.empty() && !ed.context->is_group_execution_cancelled() ); + } + } + template + bool check_for_demand(Task& t) { + if ( pass == my_delay ) { + if ( self().my_divisor > 1 ) // produce affinitized tasks while they have slot in array + return true; // do not do my_max_depth++ here, but be sure range_pool is splittable once more + else if ( self().my_divisor && my_max_depth ) { // make balancing task + self().my_divisor = 0; // once for each task; depth will be decreased in align_depth() + return true; + } + else if ( tree_node::is_peer_stolen(t) ) { + my_max_depth += __TBB_DEMAND_DEPTH_ADD; + return true; + } + } else if( begin == my_delay ) { + my_delay = pass; + } + return false; + } +}; + +class auto_partition_type: public dynamic_grainsize_mode > { +public: + auto_partition_type( const auto_partitioner& ) { + my_divisor *= __TBB_INITIAL_CHUNKS; + } + auto_partition_type( auto_partition_type& src, split) + : dynamic_grainsize_mode >(src, split()) {} + bool is_divisible() { // part of old should_execute_range() + if( my_divisor > 1 ) return true; + if( my_divisor && my_max_depth ) { // can split the task. TODO: on-stack flag instead + // keep same fragmentation while splitting for the local task pool + my_max_depth--; + my_divisor = 0; // decrease max_depth once per task + return true; + } else return false; + } + template + bool check_for_demand(Task& t) { + if (tree_node::is_peer_stolen(t)) { + my_max_depth += __TBB_DEMAND_DEPTH_ADD; + return true; + } else return false; + } + void spawn_task(task& t, task_group_context& ctx) { + spawn(t, ctx); + } +}; + +class simple_partition_type: public partition_type_base { +public: + simple_partition_type( const simple_partitioner& ) {} + simple_partition_type( const simple_partition_type&, split ) {} + //! simplified algorithm + template + void execute(StartType &start, Range &range, execution_data& ed) { + split_type split_obj = split(); // start.offer_work accepts split_type as reference + while( range.is_divisible() ) + start.offer_work( split_obj, ed ); + start.run_body( range ); + } + void spawn_task(task& t, task_group_context& ctx) { + spawn(t, ctx); + } +}; + +class static_partition_type : public linear_affinity_mode { +public: + typedef detail::proportional_split split_type; + static_partition_type( const static_partitioner& ) {} + static_partition_type( static_partition_type& p, const proportional_split& split_obj ) + : linear_affinity_mode(p, split_obj) {} +}; + +class affinity_partition_type : public dynamic_grainsize_mode > { + static const unsigned factor_power = 4; // TODO: get a unified formula based on number of computing units + slot_id* my_array; +public: + static const unsigned factor = 1 << factor_power; // number of slots in affinity array per task + typedef detail::proportional_split split_type; + affinity_partition_type( affinity_partitioner_base& ap ) { + __TBB_ASSERT( (factor&(factor-1))==0, "factor must be power of two" ); + ap.resize(factor); + my_array = ap.my_array; + my_max_depth = factor_power + 1; + __TBB_ASSERT( my_max_depth < __TBB_RANGE_POOL_CAPACITY, nullptr ); + } + affinity_partition_type(affinity_partition_type& p, split) + : dynamic_grainsize_mode >(p, split()) + , my_array(p.my_array) {} + affinity_partition_type(affinity_partition_type& p, const proportional_split& split_obj) + : dynamic_grainsize_mode >(p, split_obj) + , my_array(p.my_array) {} + void note_affinity(slot_id id) { + if( my_divisor ) + my_array[my_head] = id; + } + void spawn_task(task& t, task_group_context& ctx) { + if (my_divisor) { + if (!my_array[my_head]) { + // TODO: consider new ideas with my_array for both affinity and static partitioner's, then code reuse + spawn(t, ctx, slot_id(my_head / factor)); + } else { + spawn(t, ctx, my_array[my_head]); + } + } else { + spawn(t, ctx); + } + } +}; + +//! A simple partitioner +/** Divides the range until the range is not divisible. + @ingroup algorithms */ +class simple_partitioner { +public: + simple_partitioner() {} +private: + template friend struct start_for; + template friend struct start_reduce; + template friend struct start_deterministic_reduce; + template friend struct start_scan; + // new implementation just extends existing interface + typedef simple_partition_type task_partition_type; + // TODO: consider to make split_type public + typedef simple_partition_type::split_type split_type; + + // for parallel_scan only + class partition_type { + public: + bool should_execute_range(const execution_data& ) {return false;} + partition_type( const simple_partitioner& ) {} + partition_type( const partition_type&, split ) {} + }; +}; + +//! An auto partitioner +/** The range is initial divided into several large chunks. + Chunks are further subdivided into smaller pieces if demand detected and they are divisible. + @ingroup algorithms */ +class auto_partitioner { +public: + auto_partitioner() {} + +private: + template friend struct start_for; + template friend struct start_reduce; + template friend struct start_deterministic_reduce; + template friend struct start_scan; + // new implementation just extends existing interface + typedef auto_partition_type task_partition_type; + // TODO: consider to make split_type public + typedef auto_partition_type::split_type split_type; + + //! Backward-compatible partition for auto and affinity partition objects. + class partition_type { + size_t num_chunks; + static const size_t VICTIM_CHUNKS = 4; + public: + bool should_execute_range(const execution_data& ed) { + if( num_chunks friend struct start_for; + template friend struct start_reduce; + template friend struct start_deterministic_reduce; + template friend struct start_scan; + // new implementation just extends existing interface + typedef static_partition_type task_partition_type; + // TODO: consider to make split_type public + typedef static_partition_type::split_type split_type; +}; + +//! An affinity partitioner +class affinity_partitioner : affinity_partitioner_base { +public: + affinity_partitioner() {} + +private: + template friend struct start_for; + template friend struct start_reduce; + template friend struct start_deterministic_reduce; + template friend struct start_scan; + // new implementation just extends existing interface + typedef affinity_partition_type task_partition_type; + // TODO: consider to make split_type public + typedef affinity_partition_type::split_type split_type; +}; + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +// Partitioners +using detail::d1::auto_partitioner; +using detail::d1::simple_partitioner; +using detail::d1::static_partitioner; +using detail::d1::affinity_partitioner; +// Split types +using detail::split; +using detail::proportional_split; +} // namespace v1 + +} // namespace tbb + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) + #pragma warning (pop) +#endif // warning 4244 is back + +#undef __TBB_INITIAL_CHUNKS +#undef __TBB_RANGE_POOL_CAPACITY +#undef __TBB_INIT_DEPTH + +#endif /* __TBB_partitioner_H */ diff --git a/src/3rd party/oneapi/tbb/profiling.h b/src/3rd party/oneapi/tbb/profiling.h new file mode 100644 index 0000000000..412b5a35eb --- /dev/null +++ b/src/3rd party/oneapi/tbb/profiling.h @@ -0,0 +1,243 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_profiling_H +#define __TBB_profiling_H + +#include "detail/_config.h" +#include + +#include + +namespace tbb { +namespace detail { +inline namespace d0 { + // include list of index names + #define TBB_STRING_RESOURCE(index_name,str) index_name, + enum string_resource_index : std::uintptr_t { + #include "detail/_string_resource.h" + NUM_STRINGS + }; + #undef TBB_STRING_RESOURCE + + enum itt_relation + { + __itt_relation_is_unknown = 0, + __itt_relation_is_dependent_on, /**< "A is dependent on B" means that A cannot start until B completes */ + __itt_relation_is_sibling_of, /**< "A is sibling of B" means that A and B were created as a group */ + __itt_relation_is_parent_of, /**< "A is parent of B" means that A created B */ + __itt_relation_is_continuation_of, /**< "A is continuation of B" means that A assumes the dependencies of B */ + __itt_relation_is_child_of, /**< "A is child of B" means that A was created by B (inverse of is_parent_of) */ + __itt_relation_is_continued_by, /**< "A is continued by B" means that B assumes the dependencies of A (inverse of is_continuation_of) */ + __itt_relation_is_predecessor_to /**< "A is predecessor to B" means that B cannot start until A completes (inverse of is_dependent_on) */ + }; + +//! Unicode support +#if (_WIN32||_WIN64) + //! Unicode character type. Always wchar_t on Windows. + using tchar = wchar_t; +#else /* !WIN */ + using tchar = char; +#endif /* !WIN */ + +} // namespace d0 +} // namespace detail +} // namespace tbb + +#include +#if _WIN32||_WIN64 +#include /* mbstowcs_s */ +#endif +// Need these to work regardless of tools support +namespace tbb { +namespace detail { +namespace d1 { + enum notify_type {prepare=0, cancel, acquired, releasing, destroy}; + enum itt_domain_enum { ITT_DOMAIN_FLOW=0, ITT_DOMAIN_MAIN=1, ITT_DOMAIN_ALGO=2, ITT_NUM_DOMAINS }; +} // namespace d1 + +namespace r1 { + TBB_EXPORT void __TBB_EXPORTED_FUNC call_itt_notify(int t, void* ptr); + TBB_EXPORT void __TBB_EXPORTED_FUNC create_itt_sync(void* ptr, const tchar* objtype, const tchar* objname); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_make_task_group(d1::itt_domain_enum domain, void* group, unsigned long long group_extra, + void* parent, unsigned long long parent_extra, string_resource_index name_index); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_task_begin(d1::itt_domain_enum domain, void* task, unsigned long long task_extra, + void* parent, unsigned long long parent_extra, string_resource_index name_index); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_task_end(d1::itt_domain_enum domain); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_set_sync_name(void* obj, const tchar* name); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_metadata_str_add(d1::itt_domain_enum domain, void* addr, unsigned long long addr_extra, + string_resource_index key, const char* value); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_metadata_ptr_add(d1::itt_domain_enum domain, void* addr, unsigned long long addr_extra, + string_resource_index key, void* value); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_relation_add(d1::itt_domain_enum domain, void* addr0, unsigned long long addr0_extra, + itt_relation relation, void* addr1, unsigned long long addr1_extra); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_region_begin(d1::itt_domain_enum domain, void* region, unsigned long long region_extra, + void* parent, unsigned long long parent_extra, string_resource_index /* name_index */); + TBB_EXPORT void __TBB_EXPORTED_FUNC itt_region_end(d1::itt_domain_enum domain, void* region, unsigned long long region_extra); +} // namespace r1 + +namespace d1 { +#if TBB_USE_PROFILING_TOOLS && (_WIN32||_WIN64) + inline std::size_t multibyte_to_widechar(wchar_t* wcs, const char* mbs, std::size_t bufsize) { + std::size_t len; + mbstowcs_s(&len, wcs, bufsize, mbs, _TRUNCATE); + return len; // mbstowcs_s counts null terminator + } +#endif + +#if TBB_USE_PROFILING_TOOLS + inline void create_itt_sync(void *ptr, const char *objtype, const char *objname) { +#if (_WIN32||_WIN64) + std::size_t len_type = multibyte_to_widechar(nullptr, objtype, 0); + wchar_t *type = new wchar_t[len_type]; + multibyte_to_widechar(type, objtype, len_type); + std::size_t len_name = multibyte_to_widechar(nullptr, objname, 0); + wchar_t *name = new wchar_t[len_name]; + multibyte_to_widechar(name, objname, len_name); +#else // WIN + const char *type = objtype; + const char *name = objname; +#endif + r1::create_itt_sync(ptr, type, name); + +#if (_WIN32||_WIN64) + delete[] type; + delete[] name; +#endif // WIN + } + +// Distinguish notifications on task for reducing overheads +#if TBB_USE_PROFILING_TOOLS == 2 + inline void call_itt_task_notify(d1::notify_type t, void *ptr) { + r1::call_itt_notify(static_cast(t), ptr); + } +#else + inline void call_itt_task_notify(d1::notify_type, void *) {} +#endif // TBB_USE_PROFILING_TOOLS + + inline void call_itt_notify(d1::notify_type t, void *ptr) { + r1::call_itt_notify(static_cast(t), ptr); + } + +#if (_WIN32||_WIN64) && !__MINGW32__ + inline void itt_set_sync_name(void* obj, const wchar_t* name) { + r1::itt_set_sync_name(obj, name); + } + inline void itt_set_sync_name(void* obj, const char* name) { + std::size_t len_name = multibyte_to_widechar(nullptr, name, 0); + wchar_t *obj_name = new wchar_t[len_name]; + multibyte_to_widechar(obj_name, name, len_name); + r1::itt_set_sync_name(obj, obj_name); + delete[] obj_name; + } +#else + inline void itt_set_sync_name( void* obj, const char* name) { + r1::itt_set_sync_name(obj, name); + } +#endif //WIN + + inline void itt_make_task_group(itt_domain_enum domain, void* group, unsigned long long group_extra, + void* parent, unsigned long long parent_extra, string_resource_index name_index) { + r1::itt_make_task_group(domain, group, group_extra, parent, parent_extra, name_index); + } + + inline void itt_metadata_str_add( itt_domain_enum domain, void *addr, unsigned long long addr_extra, + string_resource_index key, const char *value ) { + r1::itt_metadata_str_add( domain, addr, addr_extra, key, value ); + } + + inline void register_node_addr(itt_domain_enum domain, void *addr, unsigned long long addr_extra, + string_resource_index key, void *value) { + r1::itt_metadata_ptr_add(domain, addr, addr_extra, key, value); + } + + inline void itt_relation_add( itt_domain_enum domain, void *addr0, unsigned long long addr0_extra, + itt_relation relation, void *addr1, unsigned long long addr1_extra ) { + r1::itt_relation_add( domain, addr0, addr0_extra, relation, addr1, addr1_extra ); + } + + inline void itt_task_begin( itt_domain_enum domain, void *task, unsigned long long task_extra, + void *parent, unsigned long long parent_extra, string_resource_index name_index ) { + r1::itt_task_begin( domain, task, task_extra, parent, parent_extra, name_index ); + } + + inline void itt_task_end( itt_domain_enum domain ) { + r1::itt_task_end( domain ); + } + + inline void itt_region_begin( itt_domain_enum domain, void *region, unsigned long long region_extra, + void *parent, unsigned long long parent_extra, string_resource_index name_index ) { + r1::itt_region_begin( domain, region, region_extra, parent, parent_extra, name_index ); + } + + inline void itt_region_end( itt_domain_enum domain, void *region, unsigned long long region_extra ) { + r1::itt_region_end( domain, region, region_extra ); + } +#else + inline void create_itt_sync(void* /*ptr*/, const char* /*objtype*/, const char* /*objname*/) {} + + inline void call_itt_notify(notify_type /*t*/, void* /*ptr*/) {} + + inline void call_itt_task_notify(notify_type /*t*/, void* /*ptr*/) {} +#endif // TBB_USE_PROFILING_TOOLS + +#if TBB_USE_PROFILING_TOOLS && !(TBB_USE_PROFILING_TOOLS == 2) +class event { +/** This class supports user event traces through itt. + Common use-case is tagging data flow graph tasks (data-id) + and visualization by Intel Advisor Flow Graph Analyzer (FGA) **/ +// TODO: Replace implementation by itt user event api. + + const std::string my_name; + + static void emit_trace(const std::string &input) { + itt_metadata_str_add( ITT_DOMAIN_FLOW, nullptr, FLOW_NULL, USER_EVENT, ( "FGA::DATAID::" + input ).c_str() ); + } + +public: + event(const std::string &input) + : my_name( input ) + { } + + void emit() { + emit_trace(my_name); + } + + static void emit(const std::string &description) { + emit_trace(description); + } + +}; +#else // TBB_USE_PROFILING_TOOLS && !(TBB_USE_PROFILING_TOOLS == 2) +// Using empty struct if user event tracing is disabled: +struct event { + event(const std::string &) { } + + void emit() { } + + static void emit(const std::string &) { } +}; +#endif // TBB_USE_PROFILING_TOOLS && !(TBB_USE_PROFILING_TOOLS == 2) +} // namespace d1 +} // namespace detail + +namespace profiling { + using detail::d1::event; +} +} // namespace tbb + + +#endif /* __TBB_profiling_H */ diff --git a/src/3rd party/oneapi/tbb/queuing_mutex.h b/src/3rd party/oneapi/tbb/queuing_mutex.h new file mode 100644 index 0000000000..bac3bb7bc2 --- /dev/null +++ b/src/3rd party/oneapi/tbb/queuing_mutex.h @@ -0,0 +1,193 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + Copyright (c) 2025 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_queuing_mutex_H +#define __TBB_queuing_mutex_H + +#include "detail/_namespace_injection.h" +#include "detail/_assert.h" +#include "detail/_utils.h" +#include "detail/_mutex_common.h" + +#include "profiling.h" + +#include + +namespace tbb { +namespace detail { +namespace d1 { + +//! Queuing mutex with local-only spinning. +/** @ingroup synchronization */ +class queuing_mutex { +public: + //! Construct unacquired mutex. + queuing_mutex() noexcept { + create_itt_sync(this, "tbb::queuing_mutex", ""); + }; + + queuing_mutex(const queuing_mutex&) = delete; + queuing_mutex& operator=(const queuing_mutex&) = delete; + + //! The scoped locking pattern + /** It helps to avoid the common problem of forgetting to release lock. + It also nicely provides the "node" for queuing locks. */ + class scoped_lock { + //! Reset fields to mean "no lock held". + void reset() { + m_mutex = nullptr; + } + + public: + //! Construct lock that has not acquired a mutex. + /** Equivalent to zero-initialization of *this. */ + scoped_lock() = default; + + //! Acquire lock on given mutex. + scoped_lock(queuing_mutex& m) { + acquire(m); + } + + //! Release lock (if lock is held). + ~scoped_lock() { + if (m_mutex) release(); + } + + //! No Copy + scoped_lock( const scoped_lock& ) = delete; + scoped_lock& operator=( const scoped_lock& ) = delete; + + //! Acquire lock on given mutex. + void acquire( queuing_mutex& m ) { + __TBB_ASSERT(!m_mutex, "scoped_lock is already holding a mutex"); + + // Must set all fields before the exchange, because once the + // exchange executes, *this becomes accessible to other threads. + m_mutex = &m; + m_next.store(nullptr, std::memory_order_relaxed); + m_going.store(0U, std::memory_order_relaxed); + + // x86 compare exchange operation always has a strong fence + // "sending" the fields initialized above to other processors. + scoped_lock* pred = m.q_tail.exchange(this); + if (pred) { + call_itt_notify(prepare, &m); + __TBB_ASSERT(pred->m_next.load(std::memory_order_relaxed) == nullptr, "the predecessor has another successor!"); + + pred->m_next.store(this, std::memory_order_release); + spin_wait_while_eq(m_going, 0U); + } + call_itt_notify(acquired, &m); + + } + + //! Acquire lock on given mutex if free (i.e. non-blocking) + bool try_acquire( queuing_mutex& m ) { + __TBB_ASSERT(!m_mutex, "scoped_lock is already holding a mutex"); + + // Must set all fields before the compare_exchange_strong, because once the + // compare_exchange_strong executes, *this becomes accessible to other threads. + m_next.store(nullptr, std::memory_order_relaxed); + m_going.store(0U, std::memory_order_relaxed); + + scoped_lock* expected = nullptr; + // The compare_exchange_strong must have release semantics, because we are + // "sending" the fields initialized above to other processors. + // x86 compare exchange operation always has a strong fence + if (m.q_tail.load(std::memory_order_relaxed) != expected || !m.q_tail.compare_exchange_strong(expected, this, std::memory_order_acq_rel)) + return false; + + m_mutex = &m; + + call_itt_notify(acquired, &m); + return true; + } + + //! Release lock. + void release() + { + __TBB_ASSERT(this->m_mutex, "no lock acquired"); + + call_itt_notify(releasing, this->m_mutex); + + if (m_next.load(std::memory_order_relaxed) == nullptr) { + scoped_lock* expected = this; + if (m_mutex->q_tail.compare_exchange_strong(expected, nullptr)) { + // this was the only item in the queue, and the queue is now empty. + reset(); + return; + } + // Someone in the queue + spin_wait_while_eq(m_next, nullptr); + } + m_next.load(std::memory_order_acquire)->m_going.store(1U, std::memory_order_release); + + reset(); + } + + private: + //! The pointer to the mutex owned, or nullptr if not holding a mutex. + queuing_mutex* m_mutex{nullptr}; + + //! The pointer to the next competitor for a mutex + std::atomic m_next{nullptr}; + + //! The local spin-wait variable + /** Inverted (0 - blocked, 1 - acquired the mutex) for the sake of + zero-initialization. Defining it as an entire word instead of + a byte seems to help performance slightly. */ + std::atomic m_going{0U}; + }; + + // Mutex traits + static constexpr bool is_rw_mutex = false; + static constexpr bool is_recursive_mutex = false; + static constexpr bool is_fair_mutex = true; + +private: + //! The last competitor requesting the lock + std::atomic q_tail{nullptr}; + +}; + +#if TBB_USE_PROFILING_TOOLS +inline void set_name(queuing_mutex& obj, const char* name) { + itt_set_sync_name(&obj, name); +} +#if (_WIN32||_WIN64) +inline void set_name(queuing_mutex& obj, const wchar_t* name) { + itt_set_sync_name(&obj, name); +} +#endif //WIN +#else +inline void set_name(queuing_mutex&, const char*) {} +#if (_WIN32||_WIN64) +inline void set_name(queuing_mutex&, const wchar_t*) {} +#endif //WIN +#endif +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::queuing_mutex; +} // namespace v1 +namespace profiling { + using detail::d1::set_name; +} +} // namespace tbb + +#endif /* __TBB_queuing_mutex_H */ diff --git a/src/3rd party/oneapi/tbb/queuing_rw_mutex.h b/src/3rd party/oneapi/tbb/queuing_rw_mutex.h new file mode 100644 index 0000000000..f8325dfd9c --- /dev/null +++ b/src/3rd party/oneapi/tbb/queuing_rw_mutex.h @@ -0,0 +1,207 @@ +/* + Copyright (c) 2005-2022 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_queuing_rw_mutex_H +#define __TBB_queuing_rw_mutex_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_assert.h" +#include "detail/_mutex_common.h" + +#include "profiling.h" + +#include +#include + +namespace tbb { +namespace detail { +namespace r1 { +struct queuing_rw_mutex_impl; +} +namespace d1 { + +//! Queuing reader-writer mutex with local-only spinning. +/** Adapted from Krieger, Stumm, et al. pseudocode at + https://www.researchgate.net/publication/221083709_A_Fair_Fast_Scalable_Reader-Writer_Lock + @ingroup synchronization */ +class queuing_rw_mutex { + friend r1::queuing_rw_mutex_impl; +public: + //! Construct unacquired mutex. + queuing_rw_mutex() noexcept { + create_itt_sync(this, "tbb::queuing_rw_mutex", ""); + } + + //! Destructor asserts if the mutex is acquired, i.e. q_tail is non-null + ~queuing_rw_mutex() { + __TBB_ASSERT(q_tail.load(std::memory_order_relaxed) == nullptr, "destruction of an acquired mutex"); + } + + //! No Copy + queuing_rw_mutex(const queuing_rw_mutex&) = delete; + queuing_rw_mutex& operator=(const queuing_rw_mutex&) = delete; + + //! The scoped locking pattern + /** It helps to avoid the common problem of forgetting to release lock. + It also nicely provides the "node" for queuing locks. */ + class scoped_lock { + friend r1::queuing_rw_mutex_impl; + //! Initialize fields to mean "no lock held". + void initialize() { + my_mutex = nullptr; + my_internal_lock.store(0, std::memory_order_relaxed); + my_going.store(0, std::memory_order_relaxed); +#if TBB_USE_ASSERT + my_state = 0xFF; // Set to invalid state + my_next.store(reinterpret_cast(reinterpret_cast(-1)), std::memory_order_relaxed); + my_prev.store(reinterpret_cast(reinterpret_cast(-1)), std::memory_order_relaxed); +#endif /* TBB_USE_ASSERT */ + } + + public: + //! Construct lock that has not acquired a mutex. + /** Equivalent to zero-initialization of *this. */ + scoped_lock() {initialize();} + + //! Acquire lock on given mutex. + scoped_lock( queuing_rw_mutex& m, bool write=true ) { + initialize(); + acquire(m,write); + } + + //! Release lock (if lock is held). + ~scoped_lock() { + if( my_mutex ) release(); + } + + //! No Copy + scoped_lock(const scoped_lock&) = delete; + scoped_lock& operator=(const scoped_lock&) = delete; + + //! Acquire lock on given mutex. + void acquire( queuing_rw_mutex& m, bool write=true ); + + //! Acquire lock on given mutex if free (i.e. non-blocking) + bool try_acquire( queuing_rw_mutex& m, bool write=true ); + + //! Release lock. + void release(); + + //! Upgrade reader to become a writer. + /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ + bool upgrade_to_writer(); + + //! Downgrade writer to become a reader. + bool downgrade_to_reader(); + + bool is_writer() const; + + private: + //! The pointer to the mutex owned, or nullptr if not holding a mutex. + queuing_rw_mutex* my_mutex; + + //! The 'pointer' to the previous and next competitors for a mutex + std::atomic my_prev; + std::atomic my_next; + + using state_t = unsigned char ; + + //! State of the request: reader, writer, active reader, other service states + std::atomic my_state; + + //! The local spin-wait variable + /** Corresponds to "spin" in the pseudocode but inverted for the sake of zero-initialization */ + std::atomic my_going; + + //! A tiny internal lock + std::atomic my_internal_lock; + }; + + // Mutex traits + static constexpr bool is_rw_mutex = true; + static constexpr bool is_recursive_mutex = false; + static constexpr bool is_fair_mutex = true; + +private: + //! The last competitor requesting the lock + std::atomic q_tail{nullptr}; +}; +#if TBB_USE_PROFILING_TOOLS +inline void set_name(queuing_rw_mutex& obj, const char* name) { + itt_set_sync_name(&obj, name); +} +#if (_WIN32||_WIN64) +inline void set_name(queuing_rw_mutex& obj, const wchar_t* name) { + itt_set_sync_name(&obj, name); +} +#endif //WIN +#else +inline void set_name(queuing_rw_mutex&, const char*) {} +#if (_WIN32||_WIN64) +inline void set_name(queuing_rw_mutex&, const wchar_t*) {} +#endif //WIN +#endif +} // namespace d1 + +namespace r1 { +TBB_EXPORT void acquire(d1::queuing_rw_mutex&, d1::queuing_rw_mutex::scoped_lock&, bool); +TBB_EXPORT bool try_acquire(d1::queuing_rw_mutex&, d1::queuing_rw_mutex::scoped_lock&, bool); +TBB_EXPORT void release(d1::queuing_rw_mutex::scoped_lock&); +TBB_EXPORT bool upgrade_to_writer(d1::queuing_rw_mutex::scoped_lock&); +TBB_EXPORT bool downgrade_to_reader(d1::queuing_rw_mutex::scoped_lock&); +TBB_EXPORT bool is_writer(const d1::queuing_rw_mutex::scoped_lock&); +} // namespace r1 + +namespace d1 { + + +inline void queuing_rw_mutex::scoped_lock::acquire(queuing_rw_mutex& m,bool write) { + r1::acquire(m, *this, write); +} + +inline bool queuing_rw_mutex::scoped_lock::try_acquire(queuing_rw_mutex& m, bool write) { + return r1::try_acquire(m, *this, write); +} + +inline void queuing_rw_mutex::scoped_lock::release() { + r1::release(*this); +} + +inline bool queuing_rw_mutex::scoped_lock::upgrade_to_writer() { + return r1::upgrade_to_writer(*this); +} + +inline bool queuing_rw_mutex::scoped_lock::downgrade_to_reader() { + return r1::downgrade_to_reader(*this); +} + +inline bool queuing_rw_mutex::scoped_lock::is_writer() const { + return r1::is_writer(*this); +} +} // namespace d1 + +} // namespace detail + +inline namespace v1 { +using detail::d1::queuing_rw_mutex; +} // namespace v1 +namespace profiling { + using detail::d1::set_name; +} +} // namespace tbb + +#endif /* __TBB_queuing_rw_mutex_H */ diff --git a/src/3rd party/oneapi/tbb/rw_mutex.h b/src/3rd party/oneapi/tbb/rw_mutex.h new file mode 100644 index 0000000000..c3fbaf657a --- /dev/null +++ b/src/3rd party/oneapi/tbb/rw_mutex.h @@ -0,0 +1,216 @@ +/* + Copyright (c) 2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_rw_mutex_H +#define __TBB_rw_mutex_H + +#include "detail/_namespace_injection.h" +#include "detail/_utils.h" +#include "detail/_waitable_atomic.h" +#include "detail/_scoped_lock.h" +#include "detail/_mutex_common.h" +#include "profiling.h" + +namespace tbb { +namespace detail { +namespace d1 { + +class rw_mutex { +public: + //! Constructors + rw_mutex() noexcept : m_state(0) { + create_itt_sync(this, "tbb::rw_mutex", ""); + } + + //! Destructor + ~rw_mutex() { + __TBB_ASSERT(!m_state.load(std::memory_order_relaxed), "destruction of an acquired mutex"); + } + + //! No Copy + rw_mutex(const rw_mutex&) = delete; + rw_mutex& operator=(const rw_mutex&) = delete; + + using scoped_lock = rw_scoped_lock; + + //! Mutex traits + static constexpr bool is_rw_mutex = true; + static constexpr bool is_recursive_mutex = false; + static constexpr bool is_fair_mutex = false; + + //! Acquire lock + void lock() { + call_itt_notify(prepare, this); + while (!try_lock()) { + if (!(m_state.load(std::memory_order_relaxed) & WRITER_PENDING)) { // no pending writers + m_state |= WRITER_PENDING; + } + + auto wakeup_condition = [&] { return !(m_state.load(std::memory_order_relaxed) & BUSY); }; + adaptive_wait_on_address(this, wakeup_condition, WRITER_CONTEXT); + } + + call_itt_notify(acquired, this); + } + + //! Try acquiring lock (non-blocking) + /** Return true if lock acquired; false otherwise. */ + bool try_lock() { + // for a writer: only possible to acquire if no active readers or writers + // Use relaxed memory fence is OK here because + // Acquire memory fence guaranteed by compare_exchange_strong() + state_type s = m_state.load(std::memory_order_relaxed); + if (!(s & BUSY)) { // no readers, no writers; mask is 1..1101 + if (m_state.compare_exchange_strong(s, WRITER)) { + call_itt_notify(acquired, this); + return true; // successfully stored writer flag + } + } + return false; + } + + //! Release lock + void unlock() { + call_itt_notify(releasing, this); + state_type curr_state = (m_state &= READERS | WRITER_PENDING); // Returns current state + + if (curr_state & WRITER_PENDING) { + r1::notify_by_address(this, WRITER_CONTEXT); + } else { + // It's possible that WRITER sleeps without WRITER_PENDING, + // because other thread might clear this bit at upgrade() + r1::notify_by_address_all(this); + } + } + + //! Lock shared ownership mutex + void lock_shared() { + call_itt_notify(prepare, this); + while (!try_lock_shared()) { + state_type has_writer = WRITER | WRITER_PENDING; + auto wakeup_condition = [&] { return !(m_state.load(std::memory_order_relaxed) & has_writer); }; + adaptive_wait_on_address(this, wakeup_condition, READER_CONTEXT); + } + __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, "invalid state of a read lock: no readers"); + } + + //! Try lock shared ownership mutex + bool try_lock_shared() { + // for a reader: acquire if no active or waiting writers + // Use relaxed memory fence is OK here because + // Acquire memory fence guaranteed by fetch_add() + state_type has_writer = WRITER | WRITER_PENDING; + if (!(m_state.load(std::memory_order_relaxed) & has_writer)) { + if (m_state.fetch_add(ONE_READER) & has_writer) { + m_state -= ONE_READER; + r1::notify_by_address(this, WRITER_CONTEXT); + } else { + call_itt_notify(acquired, this); + return true; // successfully stored increased number of readers + } + } + return false; + } + + //! Unlock shared ownership mutex + void unlock_shared() { + __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, "invalid state of a read lock: no readers"); + call_itt_notify(releasing, this); + + state_type curr_state = (m_state -= ONE_READER); // Returns current state + + if (curr_state & (WRITER_PENDING)) { + r1::notify_by_address(this, WRITER_CONTEXT); + } else { + // It's possible that WRITER sleeps without WRITER_PENDING, + // because other thread might clear this bit at upgrade() + r1::notify_by_address_all(this); + } + } + +private: + /** Internal non ISO C++ standard API **/ + //! This API is used through the scoped_lock class + + //! Upgrade reader to become a writer. + /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ + bool upgrade() { + state_type s = m_state.load(std::memory_order_relaxed); + __TBB_ASSERT(s & READERS, "invalid state before upgrade: no readers "); + // Check and set writer-pending flag. + // Required conditions: either no pending writers, or we are the only reader + // (with multiple readers and pending writer, another upgrade could have been requested) + while ((s & READERS) == ONE_READER || !(s & WRITER_PENDING)) { + if (m_state.compare_exchange_strong(s, s | WRITER | WRITER_PENDING)) { + auto wakeup_condition = [&] { return (m_state.load(std::memory_order_relaxed) & READERS) == ONE_READER; }; + while ((m_state.load(std::memory_order_relaxed) & READERS) != ONE_READER) { + adaptive_wait_on_address(this, wakeup_condition, WRITER_CONTEXT); + } + + __TBB_ASSERT((m_state.load(std::memory_order_relaxed) & (WRITER_PENDING|WRITER)) == (WRITER_PENDING | WRITER), + "invalid state when upgrading to writer"); + // Both new readers and writers are blocked at this time + m_state -= (ONE_READER + WRITER_PENDING); + return true; // successfully upgraded + } + } + // Slow reacquire + unlock_shared(); + lock(); + return false; + } + + //! Downgrade writer to a reader + void downgrade() { + __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & WRITER, nullptr), + call_itt_notify(releasing, this); + m_state += (ONE_READER - WRITER); + + if (!(m_state & WRITER_PENDING)) { + r1::notify_by_address(this, READER_CONTEXT); + } + + __TBB_ASSERT(m_state.load(std::memory_order_relaxed) & READERS, "invalid state after downgrade: no readers"); + } + + using state_type = std::intptr_t; + static constexpr state_type WRITER = 1; + static constexpr state_type WRITER_PENDING = 2; + static constexpr state_type READERS = ~(WRITER | WRITER_PENDING); + static constexpr state_type ONE_READER = 4; + static constexpr state_type BUSY = WRITER | READERS; + + using context_type = std::uintptr_t; + static constexpr context_type WRITER_CONTEXT = 0; + static constexpr context_type READER_CONTEXT = 1; + friend scoped_lock; + //! State of lock + /** Bit 0 = writer is holding lock + Bit 1 = request by a writer to acquire lock (hint to readers to wait) + Bit 2..N = number of readers holding lock */ + std::atomic m_state; +}; // class rw_mutex + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::rw_mutex; +} // namespace v1 + +} // namespace tbb + +#endif // __TBB_rw_mutex_H diff --git a/src/3rd party/oneapi/tbb/scalable_allocator.h b/src/3rd party/oneapi/tbb/scalable_allocator.h new file mode 100644 index 0000000000..31650a0a77 --- /dev/null +++ b/src/3rd party/oneapi/tbb/scalable_allocator.h @@ -0,0 +1,335 @@ +/* + Copyright (c) 2005-2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_scalable_allocator_H +#define __TBB_scalable_allocator_H + +#ifdef __cplusplus +#include "oneapi/tbb/detail/_config.h" +#include "oneapi/tbb/detail/_utils.h" +#include "oneapi/tbb/detail/_namespace_injection.h" +#include +#include +#include /* std::bad_alloc() */ +#else +#include "oneapi/tbb/detail/_export.h" +#include /* Need ptrdiff_t and size_t from here. */ +#if !defined(_MSC_VER) || defined(__clang__) +#include /* Need intptr_t from here. */ +#endif +#endif + +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#if _MSC_VER + #define __TBB_EXPORTED_FUNC __cdecl +#else + #define __TBB_EXPORTED_FUNC +#endif + +/** The "malloc" analogue to allocate block of memory of size bytes. + * @ingroup memory_allocation */ +TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_malloc(size_t size); + +/** The "free" analogue to discard a previously allocated piece of memory. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT void __TBB_EXPORTED_FUNC scalable_free(void* ptr); + +/** The "realloc" analogue complementing scalable_malloc. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_realloc(void* ptr, size_t size); + +/** The "calloc" analogue complementing scalable_malloc. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_calloc(size_t nobj, size_t size); + +/** The "posix_memalign" analogue. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_posix_memalign(void** memptr, size_t alignment, size_t size); + +/** The "_aligned_malloc" analogue. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_aligned_malloc(size_t size, size_t alignment); + +/** The "_aligned_realloc" analogue. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT void* __TBB_EXPORTED_FUNC scalable_aligned_realloc(void* ptr, size_t size, size_t alignment); + +/** The "_aligned_free" analogue. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT void __TBB_EXPORTED_FUNC scalable_aligned_free(void* ptr); + +/** The analogue of _msize/malloc_size/malloc_usable_size. + Returns the usable size of a memory block previously allocated by scalable_*, + or 0 (zero) if ptr does not point to such a block. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT size_t __TBB_EXPORTED_FUNC scalable_msize(void* ptr); + +/* Results for scalable_allocation_* functions */ +typedef enum { + TBBMALLOC_OK, + TBBMALLOC_INVALID_PARAM, + TBBMALLOC_UNSUPPORTED, + TBBMALLOC_NO_MEMORY, + TBBMALLOC_NO_EFFECT +} ScalableAllocationResult; + +/* Setting TBB_MALLOC_USE_HUGE_PAGES environment variable to 1 enables huge pages. + scalable_allocation_mode call has priority over environment variable. */ +typedef enum { + TBBMALLOC_USE_HUGE_PAGES, /* value turns using huge pages on and off */ + /* deprecated, kept for backward compatibility only */ + USE_HUGE_PAGES = TBBMALLOC_USE_HUGE_PAGES, + /* try to limit memory consumption value (Bytes), clean internal buffers + if limit is exceeded, but not prevents from requesting memory from OS */ + TBBMALLOC_SET_SOFT_HEAP_LIMIT, + /* Lower bound for the size (Bytes), that is interpreted as huge + * and not released during regular cleanup operations. */ + TBBMALLOC_SET_HUGE_SIZE_THRESHOLD +} AllocationModeParam; + +/** Set TBB allocator-specific allocation modes. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_allocation_mode(int param, intptr_t value); + +typedef enum { + /* Clean internal allocator buffers for all threads. + Returns TBBMALLOC_NO_EFFECT if no buffers cleaned, + TBBMALLOC_OK if some memory released from buffers. */ + TBBMALLOC_CLEAN_ALL_BUFFERS, + /* Clean internal allocator buffer for current thread only. + Return values same as for TBBMALLOC_CLEAN_ALL_BUFFERS. */ + TBBMALLOC_CLEAN_THREAD_BUFFERS +} ScalableAllocationCmd; + +/** Call TBB allocator-specific commands. + @ingroup memory_allocation */ +TBBMALLOC_EXPORT int __TBB_EXPORTED_FUNC scalable_allocation_command(int cmd, void *param); + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#ifdef __cplusplus + +//! The namespace rml contains components of low-level memory pool interface. +namespace rml { +class MemoryPool; + +typedef void *(*rawAllocType)(std::intptr_t pool_id, std::size_t &bytes); +// returns non-zero in case of error +typedef int (*rawFreeType)(std::intptr_t pool_id, void* raw_ptr, std::size_t raw_bytes); + +struct MemPoolPolicy { + enum { + TBBMALLOC_POOL_VERSION = 1 + }; + + rawAllocType pAlloc; + rawFreeType pFree; + // granularity of pAlloc allocations. 0 means default used. + std::size_t granularity; + int version; + // all memory consumed at 1st pAlloc call and never returned, + // no more pAlloc calls after 1st + unsigned fixedPool : 1, + // memory consumed but returned only at pool termination + keepAllMemory : 1, + reserved : 30; + + MemPoolPolicy(rawAllocType pAlloc_, rawFreeType pFree_, + std::size_t granularity_ = 0, bool fixedPool_ = false, + bool keepAllMemory_ = false) : + pAlloc(pAlloc_), pFree(pFree_), granularity(granularity_), version(TBBMALLOC_POOL_VERSION), + fixedPool(fixedPool_), keepAllMemory(keepAllMemory_), + reserved(0) {} +}; + +// enums have same values as appropriate enums from ScalableAllocationResult +// TODO: use ScalableAllocationResult in pool_create directly +enum MemPoolError { + // pool created successfully + POOL_OK = TBBMALLOC_OK, + // invalid policy parameters found + INVALID_POLICY = TBBMALLOC_INVALID_PARAM, + // requested pool policy is not supported by allocator library + UNSUPPORTED_POLICY = TBBMALLOC_UNSUPPORTED, + // lack of memory during pool creation + NO_MEMORY = TBBMALLOC_NO_MEMORY, + // action takes no effect + NO_EFFECT = TBBMALLOC_NO_EFFECT +}; + +TBBMALLOC_EXPORT MemPoolError pool_create_v1(std::intptr_t pool_id, const MemPoolPolicy *policy, + rml::MemoryPool **pool); + +TBBMALLOC_EXPORT bool pool_destroy(MemoryPool* memPool); +TBBMALLOC_EXPORT void *pool_malloc(MemoryPool* memPool, std::size_t size); +TBBMALLOC_EXPORT void *pool_realloc(MemoryPool* memPool, void *object, std::size_t size); +TBBMALLOC_EXPORT void *pool_aligned_malloc(MemoryPool* mPool, std::size_t size, std::size_t alignment); +TBBMALLOC_EXPORT void *pool_aligned_realloc(MemoryPool* mPool, void *ptr, std::size_t size, std::size_t alignment); +TBBMALLOC_EXPORT bool pool_reset(MemoryPool* memPool); +TBBMALLOC_EXPORT bool pool_free(MemoryPool *memPool, void *object); +TBBMALLOC_EXPORT MemoryPool *pool_identify(void *object); +TBBMALLOC_EXPORT std::size_t pool_msize(MemoryPool *memPool, void *object); + +} // namespace rml + +namespace tbb { +namespace detail { +namespace d1 { + +// keep throw in a separate function to prevent code bloat +template +void throw_exception(const E &e) { +#if TBB_USE_EXCEPTIONS + throw e; +#else + suppress_unused_warning(e); +#endif +} + +template +class scalable_allocator { +public: + using value_type = T; + using propagate_on_container_move_assignment = std::true_type; + + //! Always defined for TBB containers + using is_always_equal = std::true_type; + + scalable_allocator() = default; + template scalable_allocator(const scalable_allocator&) noexcept {} + + //! Allocate space for n objects. + __TBB_nodiscard T* allocate(std::size_t n) { + T* p = static_cast(scalable_malloc(n * sizeof(value_type))); + if (!p) { + throw_exception(std::bad_alloc()); + } + return p; + } + + //! Free previously allocated block of memory + void deallocate(T* p, std::size_t) { + scalable_free(p); + } + +#if TBB_ALLOCATOR_TRAITS_BROKEN + using pointer = value_type*; + using const_pointer = const value_type*; + using reference = value_type&; + using const_reference = const value_type&; + using difference_type = std::ptrdiff_t; + using size_type = std::size_t; + template struct rebind { + using other = scalable_allocator; + }; + //! Largest value for which method allocate might succeed. + size_type max_size() const noexcept { + size_type absolutemax = static_cast(-1) / sizeof (value_type); + return (absolutemax > 0 ? absolutemax : 1); + } + template + void construct(U *p, Args&&... args) + { ::new((void *)p) U(std::forward(args)...); } + void destroy(pointer p) { p->~value_type(); } + pointer address(reference x) const { return &x; } + const_pointer address(const_reference x) const { return &x; } +#endif // TBB_ALLOCATOR_TRAITS_BROKEN + +}; + +#if TBB_ALLOCATOR_TRAITS_BROKEN + template<> + class scalable_allocator { + public: + using pointer = void*; + using const_pointer = const void*; + using value_type = void; + template struct rebind { + using other = scalable_allocator; + }; + }; +#endif + +template +inline bool operator==(const scalable_allocator&, const scalable_allocator&) noexcept { return true; } + +#if !__TBB_CPP20_COMPARISONS_PRESENT +template +inline bool operator!=(const scalable_allocator&, const scalable_allocator&) noexcept { return false; } +#endif + +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT + +//! C++17 memory resource implementation for scalable allocator +//! ISO C++ Section 23.12.2 +class scalable_resource_impl : public std::pmr::memory_resource { +private: + void* do_allocate(std::size_t bytes, std::size_t alignment) override { + void* p = scalable_aligned_malloc(bytes, alignment); + if (!p) { + throw_exception(std::bad_alloc()); + } + return p; + } + + void do_deallocate(void* ptr, std::size_t /*bytes*/, std::size_t /*alignment*/) override { + scalable_free(ptr); + } + + //! Memory allocated by one instance of scalable_resource_impl could be deallocated by any + //! other instance of this class + bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override { + return this == &other || +#if __TBB_USE_OPTIONAL_RTTI + dynamic_cast(&other) != nullptr; +#else + false; +#endif + } +}; + +//! Global scalable allocator memory resource provider +inline std::pmr::memory_resource* scalable_memory_resource() noexcept { + static tbb::detail::d1::scalable_resource_impl scalable_res; + return &scalable_res; +} + +#endif // __TBB_CPP17_MEMORY_RESOURCE_PRESENT + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::scalable_allocator; +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT +using detail::d1::scalable_memory_resource; +#endif +} // namespace v1 + +} // namespace tbb + +#endif /* __cplusplus */ + +#endif /* __TBB_scalable_allocator_H */ diff --git a/src/3rd party/oneapi/tbb/spin_mutex.h b/src/3rd party/oneapi/tbb/spin_mutex.h new file mode 100644 index 0000000000..497ed9bb99 --- /dev/null +++ b/src/3rd party/oneapi/tbb/spin_mutex.h @@ -0,0 +1,138 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + Copyright (c) 2025 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_spin_mutex_H +#define __TBB_spin_mutex_H + +#include "detail/_namespace_injection.h" +#include "detail/_mutex_common.h" + +#include "profiling.h" + +#include "detail/_assert.h" +#include "detail/_utils.h" +#include "detail/_scoped_lock.h" + +#include + +namespace tbb { +namespace detail { +namespace d1 { + +#if __TBB_TSX_INTRINSICS_PRESENT +class rtm_mutex; +#endif + +/** A spin_mutex is a low-level synchronization primitive. + While locked, it causes the waiting threads to spin in a loop until the lock is released. + It should be used only for locking short critical sections + (typically less than 20 instructions) when fairness is not an issue. + If zero-initialized, the mutex is considered unheld. + @ingroup synchronization */ +class spin_mutex { +public: + //! Constructors + spin_mutex() noexcept : m_flag(false) { + create_itt_sync(this, "tbb::spin_mutex", ""); + }; + + //! Destructor + ~spin_mutex() = default; + + //! No Copy + spin_mutex(const spin_mutex&) = delete; + spin_mutex& operator=(const spin_mutex&) = delete; + + using scoped_lock = unique_scoped_lock; + + //! Mutex traits + static constexpr bool is_rw_mutex = false; + static constexpr bool is_recursive_mutex = false; + static constexpr bool is_fair_mutex = false; + + //! Acquire lock + /** Spin if the lock is taken */ + void lock() { + atomic_backoff backoff; + call_itt_notify(prepare, this); + + while (m_flag.load(std::memory_order_relaxed) || m_flag.exchange(true)) { + backoff.pause(); + } + call_itt_notify(acquired, this); + } + + //! Try acquiring lock (non-blocking) + /** Return true if lock acquired; false otherwise. */ + bool try_lock() { + bool result = !m_flag.load(std::memory_order_relaxed) && !m_flag.exchange(true); + + if (result) { + call_itt_notify(acquired, this); + } + return result; + } + + //! Release lock + void unlock() { + call_itt_notify(releasing, this); + m_flag.store(false, std::memory_order_release); + } + +protected: + std::atomic m_flag; +}; // class spin_mutex + +#if TBB_USE_PROFILING_TOOLS +inline void set_name(spin_mutex& obj, const char* name) { + itt_set_sync_name(&obj, name); +} +#if (_WIN32||_WIN64) +inline void set_name(spin_mutex& obj, const wchar_t* name) { + itt_set_sync_name(&obj, name); +} +#endif //WIN +#else +inline void set_name(spin_mutex&, const char*) {} +#if (_WIN32||_WIN64) +inline void set_name(spin_mutex&, const wchar_t*) {} +#endif // WIN +#endif +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::spin_mutex; +} // namespace v1 +namespace profiling { + using detail::d1::set_name; +} +} // namespace tbb + +#include "detail/_rtm_mutex.h" + +namespace tbb { +inline namespace v1 { +#if __TBB_TSX_INTRINSICS_PRESENT + using speculative_spin_mutex = detail::d1::rtm_mutex; +#else + using speculative_spin_mutex = detail::d1::spin_mutex; +#endif +} +} + +#endif /* __TBB_spin_mutex_H */ diff --git a/src/3rd party/oneapi/tbb/spin_rw_mutex.h b/src/3rd party/oneapi/tbb/spin_rw_mutex.h new file mode 100644 index 0000000000..3fdae3500a --- /dev/null +++ b/src/3rd party/oneapi/tbb/spin_rw_mutex.h @@ -0,0 +1,229 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_spin_rw_mutex_H +#define __TBB_spin_rw_mutex_H + +#include "detail/_namespace_injection.h" +#include "detail/_mutex_common.h" + +#include "profiling.h" + +#include "detail/_assert.h" +#include "detail/_utils.h" +#include "detail/_scoped_lock.h" + +#include + +namespace tbb { +namespace detail { +namespace d1 { + +#if __TBB_TSX_INTRINSICS_PRESENT +class rtm_rw_mutex; +#endif + +//! Fast, unfair, spinning reader-writer lock with backoff and writer-preference +/** @ingroup synchronization */ +class spin_rw_mutex { +public: + //! Constructors + spin_rw_mutex() noexcept : m_state(0) { + create_itt_sync(this, "tbb::spin_rw_mutex", ""); + } + + //! Destructor + ~spin_rw_mutex() { + __TBB_ASSERT(!m_state, "destruction of an acquired mutex"); + } + + //! No Copy + spin_rw_mutex(const spin_rw_mutex&) = delete; + spin_rw_mutex& operator=(const spin_rw_mutex&) = delete; + + using scoped_lock = rw_scoped_lock; + + //! Mutex traits + static constexpr bool is_rw_mutex = true; + static constexpr bool is_recursive_mutex = false; + static constexpr bool is_fair_mutex = false; + + //! Acquire lock + void lock() { + call_itt_notify(prepare, this); + for (atomic_backoff backoff; ; backoff.pause()) { + state_type s = m_state.load(std::memory_order_relaxed); + if (!(s & BUSY)) { // no readers, no writers + if (m_state.compare_exchange_strong(s, WRITER)) + break; // successfully stored writer flag + backoff.reset(); // we could be very close to complete op. + } else if (!(s & WRITER_PENDING)) { // no pending writers + m_state |= WRITER_PENDING; + } + } + call_itt_notify(acquired, this); + } + + //! Try acquiring lock (non-blocking) + /** Return true if lock acquired; false otherwise. */ + bool try_lock() { + // for a writer: only possible to acquire if no active readers or writers + state_type s = m_state.load(std::memory_order_relaxed); + if (!(s & BUSY)) { // no readers, no writers; mask is 1..1101 + if (m_state.compare_exchange_strong(s, WRITER)) { + call_itt_notify(acquired, this); + return true; // successfully stored writer flag + } + } + return false; + } + + //! Release lock + void unlock() { + call_itt_notify(releasing, this); + m_state &= READERS; + } + + //! Lock shared ownership mutex + void lock_shared() { + call_itt_notify(prepare, this); + for (atomic_backoff b; ; b.pause()) { + state_type s = m_state.load(std::memory_order_relaxed); + if (!(s & (WRITER | WRITER_PENDING))) { // no writer or write requests + state_type prev_state = m_state.fetch_add(ONE_READER); + if (!(prev_state & WRITER)) { + break; // successfully stored increased number of readers + } + // writer got there first, undo the increment + m_state -= ONE_READER; + } + } + call_itt_notify(acquired, this); + __TBB_ASSERT(m_state & READERS, "invalid state of a read lock: no readers"); + } + + //! Try lock shared ownership mutex + bool try_lock_shared() { + // for a reader: acquire if no active or waiting writers + state_type s = m_state.load(std::memory_order_relaxed); + if (!(s & (WRITER | WRITER_PENDING))) { // no writers + state_type prev_state = m_state.fetch_add(ONE_READER); + if (!(prev_state & WRITER)) { // got the lock + call_itt_notify(acquired, this); + return true; // successfully stored increased number of readers + } + // writer got there first, undo the increment + m_state -= ONE_READER; + } + return false; + } + + //! Unlock shared ownership mutex + void unlock_shared() { + __TBB_ASSERT(m_state & READERS, "invalid state of a read lock: no readers"); + call_itt_notify(releasing, this); + m_state -= ONE_READER; + } + +protected: + /** Internal non ISO C++ standard API **/ + //! This API is used through the scoped_lock class + + //! Upgrade reader to become a writer. + /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ + bool upgrade() { + state_type s = m_state.load(std::memory_order_relaxed); + __TBB_ASSERT(s & READERS, "invalid state before upgrade: no readers "); + // Check and set writer-pending flag. + // Required conditions: either no pending writers, or we are the only reader + // (with multiple readers and pending writer, another upgrade could have been requested) + while ((s & READERS) == ONE_READER || !(s & WRITER_PENDING)) { + if (m_state.compare_exchange_strong(s, s | WRITER | WRITER_PENDING)) { + atomic_backoff backoff; + while ((m_state.load(std::memory_order_relaxed) & READERS) != ONE_READER) backoff.pause(); + __TBB_ASSERT((m_state & (WRITER_PENDING|WRITER)) == (WRITER_PENDING | WRITER), "invalid state when upgrading to writer"); + // Both new readers and writers are blocked at this time + m_state -= (ONE_READER + WRITER_PENDING); + return true; // successfully upgraded + } + } + // Slow reacquire + unlock_shared(); + lock(); + return false; + } + + //! Downgrade writer to a reader + void downgrade() { + call_itt_notify(releasing, this); + m_state += (ONE_READER - WRITER); + __TBB_ASSERT(m_state & READERS, "invalid state after downgrade: no readers"); + } + + using state_type = std::intptr_t; + static constexpr state_type WRITER = 1; + static constexpr state_type WRITER_PENDING = 2; + static constexpr state_type READERS = ~(WRITER | WRITER_PENDING); + static constexpr state_type ONE_READER = 4; + static constexpr state_type BUSY = WRITER | READERS; + friend scoped_lock; + //! State of lock + /** Bit 0 = writer is holding lock + Bit 1 = request by a writer to acquire lock (hint to readers to wait) + Bit 2..N = number of readers holding lock */ + std::atomic m_state; +}; // class spin_rw_mutex + +#if TBB_USE_PROFILING_TOOLS +inline void set_name(spin_rw_mutex& obj, const char* name) { + itt_set_sync_name(&obj, name); +} +#if (_WIN32||_WIN64) +inline void set_name(spin_rw_mutex& obj, const wchar_t* name) { + itt_set_sync_name(&obj, name); +} +#endif // WIN +#else +inline void set_name(spin_rw_mutex&, const char*) {} +#if (_WIN32||_WIN64) +inline void set_name(spin_rw_mutex&, const wchar_t*) {} +#endif // WIN +#endif +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::spin_rw_mutex; +} // namespace v1 +namespace profiling { + using detail::d1::set_name; +} +} // namespace tbb + +#include "detail/_rtm_rw_mutex.h" + +namespace tbb { +inline namespace v1 { +#if __TBB_TSX_INTRINSICS_PRESENT + using speculative_spin_rw_mutex = detail::d1::rtm_rw_mutex; +#else + using speculative_spin_rw_mutex = detail::d1::spin_rw_mutex; +#endif +} +} + +#endif /* __TBB_spin_rw_mutex_H */ + diff --git a/src/3rd party/tbb/tbb_disable_exceptions.h b/src/3rd party/oneapi/tbb/task.h similarity index 50% rename from src/3rd party/tbb/tbb_disable_exceptions.h rename to src/3rd party/oneapi/tbb/task.h index 88ef43514f..82ce1df6cd 100644 --- a/src/3rd party/tbb/tbb_disable_exceptions.h +++ b/src/3rd party/oneapi/tbb/task.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,18 +14,24 @@ limitations under the License. */ -//! To disable use of exceptions, include this header before any other header file from the library. +#ifndef __TBB_task_H +#define __TBB_task_H -//! The macro that prevents use of exceptions in the library files -#undef TBB_USE_EXCEPTIONS -#define TBB_USE_EXCEPTIONS 0 +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_task.h" -//! Prevent compilers from issuing exception related warnings. -/** Note that the warnings are suppressed for all the code after this header is included. */ -#if _MSC_VER -#if __INTEL_COMPILER - #pragma warning (disable: 583) -#else - #pragma warning (disable: 4530 4577) -#endif -#endif +namespace tbb { +inline namespace v1 { +namespace task { +#if __TBB_RESUMABLE_TASKS + using detail::d1::suspend_point; + using detail::d1::resume; + using detail::d1::suspend; +#endif /* __TBB_RESUMABLE_TASKS */ + using detail::d1::current_context; +} // namespace task +} // namespace v1 +} // namespace tbb + +#endif /* __TBB_task_H */ diff --git a/src/3rd party/oneapi/tbb/task_arena.h b/src/3rd party/oneapi/tbb/task_arena.h new file mode 100644 index 0000000000..f41573a856 --- /dev/null +++ b/src/3rd party/oneapi/tbb/task_arena.h @@ -0,0 +1,741 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + Copyright (c) 2025 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_task_arena_H +#define __TBB_task_arena_H + +#include "detail/_config.h" + +#include "detail/_aligned_space.h" +#include "detail/_attach.h" +#include "detail/_exception.h" +#include "detail/_namespace_injection.h" +#include "detail/_small_object_pool.h" +#include "detail/_task.h" +#include "detail/_task_handle.h" +#include "info.h" +#include "task_group.h" + +#include + +namespace tbb { +namespace detail { + +namespace d1 { + +template +class task_arena_function : public delegate_base { + F &my_func; + aligned_space my_return_storage; + bool my_constructed{false}; + // The function should be called only once. + bool operator()() const override { + new (my_return_storage.begin()) R(my_func()); + return true; + } +public: + task_arena_function(F& f) : my_func(f) {} + // The function can be called only after operator() and only once. + R consume_result() { + my_constructed = true; + return std::move(*(my_return_storage.begin())); + } + ~task_arena_function() override { + if (my_constructed) { + my_return_storage.begin()->~R(); + } + } +}; + +template +class task_arena_function : public delegate_base { + F &my_func; + bool operator()() const override { + my_func(); + return true; + } +public: + task_arena_function(F& f) : my_func(f) {} + void consume_result() const {} + + friend class task_arena_base; +}; + +class task_arena_base; +class task_scheduler_observer; +} // namespace d1 + +namespace r1 { +class arena; +struct task_arena_impl; + +TBB_EXPORT void __TBB_EXPORTED_FUNC observe(d1::task_scheduler_observer&, bool); +TBB_EXPORT void __TBB_EXPORTED_FUNC initialize(d1::task_arena_base&); +TBB_EXPORT void __TBB_EXPORTED_FUNC terminate(d1::task_arena_base&); +TBB_EXPORT bool __TBB_EXPORTED_FUNC attach(d1::task_arena_base&); +TBB_EXPORT void __TBB_EXPORTED_FUNC execute(d1::task_arena_base&, d1::delegate_base&); +TBB_EXPORT void __TBB_EXPORTED_FUNC wait(d1::task_arena_base&); +TBB_EXPORT int __TBB_EXPORTED_FUNC max_concurrency(const d1::task_arena_base*); +TBB_EXPORT void __TBB_EXPORTED_FUNC isolate_within_arena(d1::delegate_base& d, std::intptr_t); + +TBB_EXPORT void __TBB_EXPORTED_FUNC enqueue(d1::task&, d1::task_arena_base*); +TBB_EXPORT void __TBB_EXPORTED_FUNC enqueue(d1::task&, d1::task_group_context&, d1::task_arena_base*); +TBB_EXPORT void __TBB_EXPORTED_FUNC submit(d1::task&, d1::task_group_context&, arena*, std::uintptr_t); + +#if __TBB_PREVIEW_PARALLEL_PHASE +TBB_EXPORT void __TBB_EXPORTED_FUNC enter_parallel_phase(d1::task_arena_base*, std::uintptr_t); +TBB_EXPORT void __TBB_EXPORTED_FUNC exit_parallel_phase(d1::task_arena_base*, std::uintptr_t); +#endif + +// Maintained for backwards compatibility +TBB_EXPORT d1::slot_id __TBB_EXPORTED_FUNC execution_slot(const d1::task_arena_base&); +} // namespace r1 + +namespace d2 { +inline void enqueue_impl(task_handle&& th, d1::task_arena_base* ta) { + __TBB_ASSERT(th != nullptr, "Attempt to schedule empty task_handle"); + + auto& ctx = task_handle_accessor::ctx_of(th); + + // Do not access th after release + task_handle_task* task_ptr = task_handle_accessor::release(th); +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + if (task_ptr->has_dependencies() && !task_ptr->release_dependency()) { + return; + } +#endif + r1::enqueue(*task_ptr, ctx, ta); +} +} //namespace d2 + +namespace d1 { + +__TBB_GLOBAL_VAR constexpr unsigned num_priority_levels = 3; +__TBB_GLOBAL_VAR constexpr int priority_stride = INT_MAX / (num_priority_levels + 1); + +class task_arena_base { + friend struct r1::task_arena_impl; + friend void r1::observe(d1::task_scheduler_observer&, bool); +public: + enum class priority : int { + low = 1 * priority_stride, + normal = 2 * priority_stride, + high = 3 * priority_stride + }; + +#if __TBB_PREVIEW_PARALLEL_PHASE + enum class leave_policy : int { + automatic = 0, + fast = 1 + }; +#endif + + using constraints = tbb::detail::d1::constraints; +protected: + //! Special settings + intptr_t my_version_and_traits; + + std::atomic my_initialization_state; + + //! nullptr if not currently initialized. + std::atomic my_arena; + static_assert(sizeof(std::atomic) == sizeof(r1::arena*), + "To preserve backward compatibility we need the equal size of an atomic pointer and a pointer"); + + //! Concurrency level for deferred initialization + int my_max_concurrency; + + //! Reserved slots for external threads + unsigned my_num_reserved_slots; + + //! Arena priority + priority my_priority; + + //! The NUMA node index to which the arena will be attached + numa_node_id my_numa_id; + + //! The core type index to which arena will be attached + core_type_id my_core_type; + + //! Number of threads per core + int my_max_threads_per_core; + + // Backward compatibility checks. + core_type_id core_type() const { + return (my_version_and_traits & core_type_support_flag) == core_type_support_flag ? my_core_type : automatic; + } + int max_threads_per_core() const { + return (my_version_and_traits & core_type_support_flag) == core_type_support_flag ? my_max_threads_per_core : automatic; + } + +#if __TBB_PREVIEW_PARALLEL_PHASE + leave_policy get_leave_policy() const { + return (my_version_and_traits & fast_leave_policy_flag) ? leave_policy::fast : leave_policy::automatic; + } + + int leave_policy_trait(leave_policy lp) const { + return lp == leave_policy::fast ? fast_leave_policy_flag : 0; + } + + void set_leave_policy(leave_policy lp) { + my_version_and_traits |= leave_policy_trait(lp); + } +#endif + + enum { + default_flags = 0, + core_type_support_flag = 1, + fast_leave_policy_flag = 1 << 1 + }; + + task_arena_base(int max_concurrency, unsigned reserved_slots, priority a_priority +#if __TBB_PREVIEW_PARALLEL_PHASE + , leave_policy lp +#endif + ) + : my_version_and_traits(default_flags | core_type_support_flag +#if __TBB_PREVIEW_PARALLEL_PHASE + | leave_policy_trait(lp) +#endif + ) + , my_initialization_state(do_once_state::uninitialized) + , my_arena(nullptr) + , my_max_concurrency(max_concurrency) + , my_num_reserved_slots(reserved_slots) + , my_priority(a_priority) + , my_numa_id(automatic) + , my_core_type(automatic) + , my_max_threads_per_core(automatic) + {} + + task_arena_base(const constraints& constraints_, unsigned reserved_slots, priority a_priority +#if __TBB_PREVIEW_PARALLEL_PHASE + , leave_policy lp +#endif + ) + : my_version_and_traits(default_flags | core_type_support_flag +#if __TBB_PREVIEW_PARALLEL_PHASE + | leave_policy_trait(lp) +#endif + ) + , my_initialization_state(do_once_state::uninitialized) + , my_arena(nullptr) + , my_max_concurrency(constraints_.max_concurrency) + , my_num_reserved_slots(reserved_slots) + , my_priority(a_priority) + , my_numa_id(constraints_.numa_id) + , my_core_type(constraints_.core_type) + , my_max_threads_per_core(constraints_.max_threads_per_core) + {} + +public: + //! Typedef for number of threads that is automatic. + static const int automatic = -1; + //! Typedef for current thread index in an uninitialized arena. + static const int not_initialized = -2; +#if __TBB_PREVIEW_TASK_ARENA_CORE_TYPE_SELECTOR + //! Typedef for core type(s) to be specified by the provided selector. + static const int selectable = -2; +#endif +}; + +template +R isolate_impl(F& f) { + task_arena_function func(f); + r1::isolate_within_arena(func, /*isolation*/ 0); + return func.consume_result(); +} + +template +class enqueue_task : public task { + small_object_allocator m_allocator; + const F m_func; + + void finalize(const execution_data& ed) { + m_allocator.delete_object(this, ed); + } + task* execute(execution_data& ed) override { + m_func(); + finalize(ed); + return nullptr; + } + task* cancel(execution_data&) override { + __TBB_ASSERT_RELEASE(false, "Unhandled exception from enqueue task is caught"); + return nullptr; + } +public: + enqueue_task(const F& f, small_object_allocator& alloc) : m_allocator(alloc), m_func(f) {} + enqueue_task(F&& f, small_object_allocator& alloc) : m_allocator(alloc), m_func(std::move(f)) {} +}; + +template +void enqueue_impl(F&& f, task_arena_base* ta) { + small_object_allocator alloc{}; + r1::enqueue(*alloc.new_object::type>>(std::forward(f), alloc), ta); +} +/** 1-to-1 proxy representation class of scheduler's arena + * Constructors set up settings only, real construction is deferred till the first method invocation + * Destructor only removes one of the references to the inner arena representation. + * Final destruction happens when all the references (and the work) are gone. + */ +class task_arena : public task_arena_base { + + void mark_initialized() { + __TBB_ASSERT( my_arena.load(std::memory_order_relaxed), "task_arena initialization is incomplete" ); + my_initialization_state.store(do_once_state::initialized, std::memory_order_release); + } + + template + R execute_impl(F& f) { + initialize(); + task_arena_function func(f); + r1::execute(*this, func); + return func.consume_result(); + } + + d2::task_group_status wait_for_impl(d2::task_group& tg) { + d2::task_group_status status = d2::task_group_status::not_complete; + d2::wait_delegate wd{tg, status}; + r1::execute(*this, wd); + __TBB_ASSERT(status != d2::task_group_status::not_complete, + "unexpected premature exit from wait_for: task group status is still not complete"); + return status; + } +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + d2::task_group_status wait_for_impl(d2::task_completion_handle& comp_handle) { + d2::task_group_status status = d2::task_group_status::not_complete; + d2::wait_completion_delegate wd{comp_handle, status}; + r1::execute(*this, wd); + __TBB_ASSERT(status != d2::task_group_status::not_complete, + "unexpected premature exit from wait_for: task status is still not complete"); + return status; + } +#endif + +public: + //! Creates task_arena with certain concurrency limits + /** Sets up settings only, real construction is deferred till the first method invocation + * @arg max_concurrency specifies total number of slots in arena where threads work + * @arg reserved_slots specifies number of slots to be used by external threads only. + * Value of 1 is default and reflects behavior of implicit arenas. + **/ + task_arena(int max_concurrency_ = automatic, unsigned reserved_slots = 1, + priority a_priority = priority::normal +#if __TBB_PREVIEW_PARALLEL_PHASE + , leave_policy lp = leave_policy::automatic +#endif + ) + : task_arena_base(max_concurrency_, reserved_slots, a_priority +#if __TBB_PREVIEW_PARALLEL_PHASE + , lp +#endif + ) + {} + + //! Creates task arena pinned to certain NUMA node + task_arena(const constraints& constraints_, unsigned reserved_slots = 1, + priority a_priority = priority::normal +#if __TBB_PREVIEW_PARALLEL_PHASE + , leave_policy lp = leave_policy::automatic +#endif + ) + : task_arena_base(constraints_, reserved_slots, a_priority +#if __TBB_PREVIEW_PARALLEL_PHASE + , lp +#endif + ) + {} + +#if __TBB_PREVIEW_TASK_ARENA_CORE_TYPE_SELECTOR + //! Creates task arena with a custom selector for core types + template (std::declval()(std::declval>())))> + task_arena(const constraints& constraints_, Selector selector_, + unsigned reserved_for_masters = 1, priority a_priority = priority::normal +#if __TBB_PREVIEW_PARALLEL_PHASE + , leave_policy lp = leave_policy::automatic +#endif + ) + : task_arena_base(constraints_, reserved_for_masters, a_priority +#if __TBB_PREVIEW_PARALLEL_PHASE + , lp +#endif + ) + { + if (my_core_type == selectable) { + my_core_type = apply_core_type_selector(selector_); + } + } +#endif + + //! Copies settings from another task_arena + task_arena(const task_arena& a) // copy settings but not the reference or instance + : task_arena_base( + constraints{} + .set_numa_id(a.my_numa_id) + .set_max_concurrency(a.my_max_concurrency) + .set_core_type(a.my_core_type) + .set_max_threads_per_core(a.my_max_threads_per_core) + , a.my_num_reserved_slots, a.my_priority +#if __TBB_PREVIEW_PARALLEL_PHASE + , a.get_leave_policy() +#endif + ) + + {} + + //! Tag class used to indicate the "attaching" constructor + struct attach {}; + + //! Creates an instance of task_arena attached to the current arena of the thread + explicit task_arena( attach ) + : task_arena_base(automatic, 1, priority::normal +#if __TBB_PREVIEW_PARALLEL_PHASE + , leave_policy::automatic +#endif + ) // use default settings if attach fails + { + if (r1::attach(*this)) { + mark_initialized(); + } + } + + //! Creates an instance of task_arena attached to the current arena of the thread + explicit task_arena(d1::attach) + : task_arena(attach{}) + {} + + //! Forces allocation of the resources for the task_arena as specified in constructor arguments + void initialize() { + atomic_do_once([this]{ r1::initialize(*this); }, my_initialization_state); + } + + //! Overrides concurrency level and forces initialization of internal representation + void initialize(int max_concurrency_, unsigned reserved_slots = 1, + priority a_priority = priority::normal +#if __TBB_PREVIEW_PARALLEL_PHASE + , leave_policy lp = leave_policy::automatic +#endif + ) + { + __TBB_ASSERT(!my_arena.load(std::memory_order_relaxed), "Impossible to modify settings of an already initialized task_arena"); + if( !is_active() ) { + my_max_concurrency = max_concurrency_; + my_num_reserved_slots = reserved_slots; + my_priority = a_priority; +#if __TBB_PREVIEW_PARALLEL_PHASE + set_leave_policy(lp); +#endif + r1::initialize(*this); + mark_initialized(); + } + } + + //! Overrides constraints and forces initialization of internal representation + void initialize(constraints constraints_, unsigned reserved_slots = 1, + priority a_priority = priority::normal +#if __TBB_PREVIEW_PARALLEL_PHASE + , leave_policy lp = leave_policy::automatic +#endif + ) + { + __TBB_ASSERT(!my_arena.load(std::memory_order_relaxed), "Impossible to modify settings of an already initialized task_arena"); + if( !is_active() ) { + my_numa_id = constraints_.numa_id; + my_max_concurrency = constraints_.max_concurrency; + my_core_type = constraints_.core_type; + my_max_threads_per_core = constraints_.max_threads_per_core; + my_num_reserved_slots = reserved_slots; + my_priority = a_priority; +#if __TBB_PREVIEW_PARALLEL_PHASE + set_leave_policy(lp); +#endif + r1::initialize(*this); + mark_initialized(); + } + } + +#if __TBB_PREVIEW_TASK_ARENA_CORE_TYPE_SELECTOR + //! Overrides constraints with a custom selector for core types and forces initialization of internal representation + template(std::declval()(std::declval>())))> + void initialize(constraints constraints_, Selector selector_, + unsigned reserved_for_masters = 1, priority a_priority = priority::normal +#if __TBB_PREVIEW_PARALLEL_PHASE + , leave_policy lp = leave_policy::automatic +#endif + ) + { + __TBB_ASSERT(!my_arena.load(std::memory_order_relaxed), "Impossible to modify settings of an already initialized task_arena"); + if( !is_active() ) { + my_numa_id = constraints_.numa_id; + my_max_concurrency = constraints_.max_concurrency; + my_core_type = constraints_.core_type; + my_max_threads_per_core = constraints_.max_threads_per_core; + my_num_reserved_slots = reserved_for_masters; + my_priority = a_priority; +#if __TBB_PREVIEW_PARALLEL_PHASE + set_leave_policy(lp); +#endif + if (my_core_type == selectable) { + my_core_type = apply_core_type_selector(selector_); + } + r1::initialize(*this); + mark_initialized(); + } + } +#endif /*__TBB_PREVIEW_TASK_ARENA_CORE_TYPE_SELECTOR*/ + + //! Attaches this instance to the current arena of the thread + void initialize(attach) { + // TODO: decide if this call must be thread-safe + __TBB_ASSERT(!my_arena.load(std::memory_order_relaxed), "Impossible to modify settings of an already initialized task_arena"); + if( !is_active() ) { + if ( !r1::attach(*this) ) { + r1::initialize(*this); + } + mark_initialized(); + } + } + + //! Attaches this instance to the current arena of the thread + void initialize(d1::attach) { + initialize(attach{}); + } + + //! Removes the reference to the internal arena representation. + //! Not thread safe wrt concurrent invocations of other methods. + void terminate() { + if( is_active() ) { + r1::terminate(*this); + my_initialization_state.store(do_once_state::uninitialized, std::memory_order_relaxed); + } + } + + //! Removes the reference to the internal arena representation, and destroys the external object. + //! Not thread safe wrt concurrent invocations of other methods. + ~task_arena() { + terminate(); + } + + //! Returns true if the arena is active (initialized); false otherwise. + //! The name was chosen to match a task_scheduler_init method with the same semantics. + bool is_active() const { + return my_initialization_state.load(std::memory_order_acquire) == do_once_state::initialized; + } + + //! Enqueues a task into the arena to process a functor, and immediately returns. + //! Does not require the calling thread to join the arena + + template + void enqueue(F&& f) { + initialize(); + enqueue_impl(std::forward(f), this); + } + + //! Enqueues a task into the arena to process a functor wrapped in task_handle, and immediately returns. + //! Does not require the calling thread to join the arena + void enqueue(d2::task_handle&& th) { + initialize(); + d2::enqueue_impl(std::move(th), this); + } + + //! Adds a task to process a functor into the task_group and then enqueues it into the arena, + //! and immediately returns. + //! Does not require the calling thread to join the arena. + template + void enqueue(F&& f, d2::task_group& tg) { + initialize(); + d2::enqueue_impl(tg.defer(std::forward(f)), this); + } + + //! Waits for all tasks in the task group to complete or be canceled. + //! During the wait, may execute tasks in the task_arena. + d2::task_group_status wait_for(d2::task_group& tg) { + initialize(); + return wait_for_impl(tg); + } + +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + d2::task_group_status wait_for(d2::task_completion_handle& comp_handle) { + initialize(); + return wait_for_impl(comp_handle); + } +#endif + + //! Joins the arena and executes a mutable functor, then returns + //! If not possible to join, wraps the functor into a task, enqueues it and waits for task completion + //! Can decrement the arena demand for workers, causing a worker to leave and free a slot to the calling thread + //! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void). + template + auto execute(F&& f) -> decltype(f()) { + return execute_impl(f); + } + +#if __TBB_PREVIEW_PARALLEL_PHASE + void start_parallel_phase() { + initialize(); + r1::enter_parallel_phase(this, /*reserved*/0); + } + void end_parallel_phase(bool with_fast_leave = false) { + __TBB_ASSERT(my_initialization_state.load(std::memory_order_relaxed) == do_once_state::initialized, nullptr); + // It is guaranteed by the standard that conversion of boolean to integral type will result in either 0 or 1 + r1::exit_parallel_phase(this, static_cast(with_fast_leave)); + } + + class scoped_parallel_phase : no_copy { + task_arena& arena; + bool one_time_fast_leave; + public: + scoped_parallel_phase(task_arena& ta, bool with_fast_leave = false) + : arena(ta), one_time_fast_leave(with_fast_leave) + { + arena.start_parallel_phase(); + } + ~scoped_parallel_phase() { + arena.end_parallel_phase(one_time_fast_leave); + } + }; +#endif + +#if __TBB_EXTRA_DEBUG + //! Returns my_num_reserved_slots + int debug_reserved_slots() const { + // Handle special cases inside the library + return my_num_reserved_slots; + } + + //! Returns my_max_concurrency + int debug_max_concurrency() const { + // Handle special cases inside the library + return my_max_concurrency; + } + + //! Wait for all work in the arena to be completed + //! Even submitted by other application threads + //! Joins arena if/when possible (in the same way as execute()) + void debug_wait_until_empty() { + initialize(); + r1::wait(*this); + } +#endif //__TBB_EXTRA_DEBUG + + //! Returns the maximal number of threads that can work inside the arena + int max_concurrency() const { + // Handle special cases inside the library + return (my_max_concurrency > 1) ? my_max_concurrency : r1::max_concurrency(this); + } + + friend void submit(task& t, task_arena& ta, task_group_context& ctx, bool as_critical) { + __TBB_ASSERT(ta.is_active(), nullptr); + call_itt_task_notify(releasing, &t); + r1::submit(t, ctx, ta.my_arena.load(std::memory_order_relaxed), as_critical ? 1 : 0); + } +}; + +//! Executes a mutable functor in isolation within the current task arena. +//! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void). +template +inline auto isolate(F&& f) -> decltype(f()) { + return isolate_impl(f); +} + +//! Returns the index, aka slot number, of the calling thread in its current arena +inline int current_thread_index() { + slot_id idx = r1::execution_slot(nullptr); + return idx == slot_id(-1) ? task_arena_base::not_initialized : int(idx); +} + +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS +inline bool is_inside_task() { + return nullptr != current_context(); +} +#endif //__TBB_PREVIEW_TASK_GROUP_EXTENSIONS + +//! Returns the maximal number of threads that can work inside the arena +inline int max_concurrency() { + return r1::max_concurrency(nullptr); +} + +inline void enqueue(d2::task_handle&& th) { + d2::enqueue_impl(std::move(th), nullptr); +} + +template +inline void enqueue(F&& f) { + enqueue_impl(std::forward(f), nullptr); +} + +template +inline void enqueue(F&& f, d2::task_group& tg) { + d2::enqueue_impl(tg.defer(std::forward(f)), nullptr); +} + +#if __TBB_PREVIEW_PARALLEL_PHASE +inline void start_parallel_phase() { + r1::enter_parallel_phase(nullptr, /*reserved*/0); +} + +inline void end_parallel_phase(bool with_fast_leave) { + // It is guaranteed by the standard that conversion of boolean to integral type will result in either 0 or 1 + r1::exit_parallel_phase(nullptr, static_cast(with_fast_leave)); +} +#endif + +inline std::vector create_numa_task_arenas(d1::constraints c = {}, + unsigned reserved_slots = 0) +{ + static std::vector node_indices = d1::numa_nodes(); + std::vector numa_arenas; + numa_arenas.reserve(node_indices.size()); + for (auto numa_id : node_indices) { + numa_arenas.emplace_back(c.set_numa_id(numa_id), reserved_slots); + } + return numa_arenas; +} + +using r1::submit; + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::task_arena; +using detail::d1::attach; +using detail::d1::create_numa_task_arenas; + +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS +using detail::d1::is_inside_task; +#endif + +namespace this_task_arena { +using detail::d1::current_thread_index; +using detail::d1::max_concurrency; +using detail::d1::isolate; + +using detail::d1::enqueue; + +#if __TBB_PREVIEW_PARALLEL_PHASE +using detail::d1::start_parallel_phase; +using detail::d1::end_parallel_phase; +#endif +} // namespace this_task_arena + +} // inline namespace v1 + +} // namespace tbb +#endif /* __TBB_task_arena_H */ diff --git a/src/3rd party/oneapi/tbb/task_group.h b/src/3rd party/oneapi/tbb/task_group.h new file mode 100644 index 0000000000..7d647310b7 --- /dev/null +++ b/src/3rd party/oneapi/tbb/task_group.h @@ -0,0 +1,841 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + Copyright (c) 2025 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_task_group_H +#define __TBB_task_group_H + +#include "detail/_config.h" +#include "detail/_namespace_injection.h" +#include "detail/_assert.h" +#include "detail/_utils.h" +#include "detail/_template_helpers.h" +#include "detail/_exception.h" +#include "detail/_task.h" +#include "detail/_small_object_pool.h" +#include "detail/_intrusive_list_node.h" +#include "detail/_task_handle.h" + +#include "profiling.h" + +#include + +#if _MSC_VER && !defined(__INTEL_COMPILER) + // Suppress warning: structure was padded due to alignment specifier + #pragma warning(push) + #pragma warning(disable:4324) +#endif + +namespace tbb { +namespace detail { + +namespace d1 { +class delegate_base; +class task_arena_base; +class task_group_context; +} + +namespace r1 { +// Forward declarations +class tbb_exception_ptr; +class cancellation_disseminator; +class thread_data; +class task_dispatcher; +template +class context_guard_helper; +struct task_arena_impl; +class context_list; +void handle_context_exception(d1::task_group_context& ctx, bool rethrow); + +TBB_EXPORT void __TBB_EXPORTED_FUNC execute(d1::task_arena_base&, d1::delegate_base&); +TBB_EXPORT void __TBB_EXPORTED_FUNC isolate_within_arena(d1::delegate_base&, std::intptr_t); + +TBB_EXPORT void __TBB_EXPORTED_FUNC initialize(d1::task_group_context&); +TBB_EXPORT void __TBB_EXPORTED_FUNC destroy(d1::task_group_context&); +TBB_EXPORT void __TBB_EXPORTED_FUNC reset(d1::task_group_context&); +TBB_EXPORT bool __TBB_EXPORTED_FUNC cancel_group_execution(d1::task_group_context&); +TBB_EXPORT bool __TBB_EXPORTED_FUNC is_group_execution_cancelled(d1::task_group_context&); +TBB_EXPORT void __TBB_EXPORTED_FUNC capture_fp_settings(d1::task_group_context&); + +struct task_group_context_impl; +} + +namespace d2 { + +template +d1::task* task_ptr_or_nullptr(F&& f); + +template +class function_task : public task_handle_task { + //TODO: apply empty base optimization here + const F m_func; + +private: + static void destroy_function_task(task_handle_task* p, d1::small_object_allocator& alloc, + const d1::execution_data* ed) + { + if (ed) { + alloc.delete_object(static_cast(p), *ed); + } else { + alloc.delete_object(static_cast(p)); + } + } + + d1::task* execute(d1::execution_data& ed) override { + __TBB_ASSERT(ed.context == &this->ctx(), "The task group context should be used for all tasks"); + task* next_task = task_ptr_or_nullptr(m_func); +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + task_handle_task* successor_task = this->complete_and_try_get_successor(); + + if (next_task != nullptr) { + // If there are both task returned from the body and the successor task + // Bypassing the body task and spawning the successor one + if (successor_task != nullptr) d1::spawn(*successor_task, successor_task->ctx()); + } else { + next_task = successor_task; + } +#endif + this->destroy(&ed); + return next_task; + } + d1::task* cancel(d1::execution_data& ed) override { + task* task_ptr = nullptr; +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + // TODO: complete_and_try_get_successor returns one ready successor task, others are spawned and cancelled by the scheduler + // Should cancel() be called directly instead? + task_ptr = this->cancel_and_try_get_successor(); +#endif + this->destroy(&ed); + return task_ptr; + } +public: + template + function_task(FF&& f, d1::wait_tree_vertex_interface* vertex, d1::task_group_context& ctx, d1::small_object_allocator& alloc) + : task_handle_task{vertex, ctx, alloc, destroy_function_task} + , m_func(std::forward(f)) {} +}; + +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + template + d1::task* task_ptr_or_nullptr_impl(std::false_type, F&& f){ + task_handle th = std::forward(f)(); + task_handle_task* task_ptr = task_handle_accessor::release(th); + // If task has unresolved dependencies, it can't be bypassed + if (task_ptr && task_ptr->has_dependencies() && !task_ptr->release_dependency()) { + task_ptr = nullptr; + } + + return task_ptr; + } + + template + d1::task* task_ptr_or_nullptr_impl(std::true_type, F&& f){ + std::forward(f)(); + return nullptr; + } + + template + d1::task* task_ptr_or_nullptr(F&& f){ + using is_void_t = std::is_void< + decltype(std::forward(f)()) + >; + + return task_ptr_or_nullptr_impl(is_void_t{}, std::forward(f)); + } +#else + template + d1::task* task_ptr_or_nullptr(F&& f){ + std::forward(f)(); + return nullptr; + } +#endif // __TBB_PREVIEW_TASK_GROUP_EXTENSIONS +} // namespace d2 + +namespace d1 { + +// This structure is left here for backward compatibility check +struct context_list_node { + std::atomic prev{}; + std::atomic next{}; +}; + +//! Used to form groups of tasks +/** @ingroup task_scheduling + The context services explicit cancellation requests from user code, and unhandled + exceptions intercepted during tasks execution. Intercepting an exception results + in generating internal cancellation requests (which is processed in exactly the + same way as external ones). + + The context is associated with one or more root tasks and defines the cancellation + group that includes all the descendants of the corresponding root task(s). Association + is established when a context object is passed as an argument to the task::allocate_root() + method. See task_group_context::task_group_context for more details. + + The context can be bound to another one, and other contexts can be bound to it, + forming a tree-like structure: parent -> this -> children. Arrows here designate + cancellation propagation direction. If a task in a cancellation group is cancelled + all the other tasks in this group and groups bound to it (as children) get cancelled too. +**/ +class task_group_context : no_copy { +public: + enum traits_type { + fp_settings = 1 << 1, + concurrent_wait = 1 << 2, + default_traits = 0 + }; + enum kind_type { + isolated, + bound + }; +private: + //! Space for platform-specific FPU settings. + /** Must only be accessed inside TBB binaries, and never directly in user + code or inline methods. */ + std::uint64_t my_cpu_ctl_env; + + //! Specifies whether cancellation was requested for this task group. + std::atomic my_cancellation_requested; + + //! Versioning for run-time checks and behavioral traits of the context. + enum class task_group_context_version : std::uint8_t { + unused = 1 // ensure that new versions, if any, will not clash with previously used ones + }; + task_group_context_version my_version; + + //! The context traits. + struct context_traits { + bool fp_settings : 1; + bool concurrent_wait : 1; + bool bound : 1; + bool reserved1 : 1; + bool reserved2 : 1; + bool reserved3 : 1; + bool reserved4 : 1; + bool reserved5 : 1; + } my_traits; + + static_assert(sizeof(context_traits) == 1, "Traits shall fit into one byte."); + + static constexpr std::uint8_t may_have_children = 1; + //! The context internal state (currently only may_have_children). + std::atomic my_may_have_children; + + enum class state : std::uint8_t { + created, + locked, + isolated, + bound, + dead, + proxy = std::uint8_t(-1) //the context is not the real one, but proxy to other one + }; + + //! The synchronization machine state to manage lifetime. + std::atomic my_state; + + union { + //! Pointer to the context of the parent cancellation group. nullptr for isolated contexts. + task_group_context* my_parent; + + //! Pointer to the actual context 'this' context represents a proxy of. + task_group_context* my_actual_context; + }; + + //! Thread data instance that registered this context in its list. + r1::context_list* my_context_list; + static_assert(sizeof(std::atomic) == sizeof(r1::context_list*), "To preserve backward compatibility these types should have the same size"); + + //! Used to form the thread specific list of contexts without additional memory allocation. + /** A context is included into the list of the current thread when its binding to + its parent happens. Any context can be present in the list of one thread only. **/ + intrusive_list_node my_node; + static_assert(sizeof(intrusive_list_node) == sizeof(context_list_node), "To preserve backward compatibility these types should have the same size"); + + //! Pointer to the container storing exception being propagated across this task group. + std::atomic my_exception; + static_assert(sizeof(std::atomic) == sizeof(r1::tbb_exception_ptr*), + "backward compatibility check"); + + //! Used to set and maintain stack stitching point for Intel Performance Tools. + void* my_itt_caller; + + //! Description of algorithm for scheduler based instrumentation. + string_resource_index my_name; + + char padding[max_nfs_size + - sizeof(std::uint64_t) // my_cpu_ctl_env + - sizeof(std::atomic) // my_cancellation_requested + - sizeof(std::uint8_t) // my_version + - sizeof(context_traits) // my_traits + - sizeof(std::atomic) // my_state + - sizeof(std::atomic) // my_state + - sizeof(task_group_context*) // my_parent + - sizeof(r1::context_list*) // my_context_list + - sizeof(intrusive_list_node) // my_node + - sizeof(std::atomic) // my_exception + - sizeof(void*) // my_itt_caller + - sizeof(string_resource_index) // my_name + ]; + + task_group_context(context_traits t, string_resource_index name) + : my_version{task_group_context_version::unused}, my_name{name} + { + my_traits = t; // GCC4.8 issues warning list initialization for bitset (missing-field-initializers) + r1::initialize(*this); + } + + task_group_context(task_group_context* actual_context) + : my_version{task_group_context_version::unused} + , my_state{state::proxy} + , my_actual_context{actual_context} + { + __TBB_ASSERT(my_actual_context, "Passed pointer value points to nothing."); + my_name = actual_context->my_name; + + // no need to initialize 'this' context as it acts as a proxy for my_actual_context, which + // initialization is a user-side responsibility. + } + + static context_traits make_traits(kind_type relation_with_parent, std::uintptr_t user_traits) { + context_traits ct; + ct.fp_settings = (user_traits & fp_settings) == fp_settings; + ct.concurrent_wait = (user_traits & concurrent_wait) == concurrent_wait; + ct.bound = relation_with_parent == bound; + ct.reserved1 = ct.reserved2 = ct.reserved3 = ct.reserved4 = ct.reserved5 = false; + return ct; + } + + bool is_proxy() const { + return my_state.load(std::memory_order_relaxed) == state::proxy; + } + + task_group_context& actual_context() noexcept { + if (is_proxy()) { + __TBB_ASSERT(my_actual_context, "Actual task_group_context is not set."); + return *my_actual_context; + } + return *this; + } + + const task_group_context& actual_context() const noexcept { + if (is_proxy()) { + __TBB_ASSERT(my_actual_context, "Actual task_group_context is not set."); + return *my_actual_context; + } + return *this; + } + +public: + //! Default & binding constructor. + /** By default a bound context is created. That is this context will be bound + (as child) to the context of the currently executing task . Cancellation + requests passed to the parent context are propagated to all the contexts + bound to it. Similarly priority change is propagated from the parent context + to its children. + + If task_group_context::isolated is used as the argument, then the tasks associated + with this context will never be affected by events in any other context. + + Creating isolated contexts involve much less overhead, but they have limited + utility. Normally when an exception occurs in an algorithm that has nested + ones running, it is desirably to have all the nested algorithms cancelled + as well. Such a behavior requires nested algorithms to use bound contexts. + + There is one good place where using isolated algorithms is beneficial. It is + an external thread. That is if a particular algorithm is invoked directly from + the external thread (not from a TBB task), supplying it with explicitly + created isolated context will result in a faster algorithm startup. + + VERSIONING NOTE: + Implementation(s) of task_group_context constructor(s) cannot be made + entirely out-of-line because the run-time version must be set by the user + code. This will become critically important for binary compatibility, if + we ever have to change the size of the context object. **/ + + task_group_context(kind_type relation_with_parent = bound, + std::uintptr_t t = default_traits) + : task_group_context(make_traits(relation_with_parent, t), CUSTOM_CTX) {} + + // Custom constructor for instrumentation of oneTBB algorithm + task_group_context(string_resource_index name ) + : task_group_context(make_traits(bound, default_traits), name) {} + + // Do not introduce any logic on user side since it might break state propagation assumptions + ~task_group_context() { + // When 'this' serves as a proxy, the initialization does not happen - nor should the + // destruction. + if (!is_proxy()) + { + r1::destroy(*this); + } + } + + //! Forcefully reinitializes the context after the task tree it was associated with is completed. + /** Because the method assumes that all the tasks that used to be associated with + this context have already finished, calling it while the context is still + in use somewhere in the task hierarchy leads to undefined behavior. + + IMPORTANT: This method is not thread safe! + + The method does not change the context's parent if it is set. **/ + void reset() { + r1::reset(actual_context()); + } + + //! Initiates cancellation of all tasks in this cancellation group and its subordinate groups. + /** \return false if cancellation has already been requested, true otherwise. + + Note that canceling never fails. When false is returned, it just means that + another thread (or this one) has already sent cancellation request to this + context or to one of its ancestors (if this context is bound). It is guaranteed + that when this method is concurrently called on the same not yet cancelled + context, true will be returned by one and only one invocation. **/ + bool cancel_group_execution() { + return r1::cancel_group_execution(actual_context()); + } + + //! Returns true if the context received cancellation request. + bool is_group_execution_cancelled() { + return r1::is_group_execution_cancelled(actual_context()); + } + +#if __TBB_FP_CONTEXT + //! Captures the current FPU control settings to the context. + /** Because the method assumes that all the tasks that used to be associated with + this context have already finished, calling it while the context is still + in use somewhere in the task hierarchy leads to undefined behavior. + + IMPORTANT: This method is not thread safe! + + The method does not change the FPU control settings of the context's parent. **/ + void capture_fp_settings() { + r1::capture_fp_settings(actual_context()); + } +#endif + + //! Returns the user visible context trait + std::uintptr_t traits() const { + std::uintptr_t t{}; + const task_group_context& ctx = actual_context(); + t |= ctx.my_traits.fp_settings ? fp_settings : 0; + t |= ctx.my_traits.concurrent_wait ? concurrent_wait : 0; + return t; + } +private: + //// TODO: cleanup friends + friend class r1::cancellation_disseminator; + friend class r1::thread_data; + friend class r1::task_dispatcher; + template + friend class r1::context_guard_helper; + friend struct r1::task_arena_impl; + friend struct r1::task_group_context_impl; + friend class d2::task_group_base; + friend void r1::handle_context_exception(d1::task_group_context&, bool rethrow); +}; // class task_group_context + +static_assert(sizeof(task_group_context) == 128, "Wrong size of task_group_context"); + +inline bool is_current_task_group_canceling() { + task_group_context* ctx = current_context(); + return ctx ? ctx->is_group_execution_cancelled() : false; +} + +} // namespace d1 + +namespace d2 { + +class task_group; +class structured_task_group; +#if TBB_PREVIEW_ISOLATED_TASK_GROUP +class isolated_task_group; +#endif + +template +class function_stack_task +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + : public d2::dynamic_state_task +#else + : public d1::task +#endif +{ + const F& m_func; + d1::wait_tree_vertex_interface* m_wait_tree_vertex; + + void finalize() { + m_wait_tree_vertex->release(); + } + task* execute(d1::execution_data&) override { + task* res = d2::task_ptr_or_nullptr(m_func); + finalize(); + return res; + } + task* cancel(d1::execution_data&) override { + finalize(); + return nullptr; + } +public: + function_stack_task(const F& f, d1::wait_tree_vertex_interface* vertex) : m_func(f), m_wait_tree_vertex(vertex) { + m_wait_tree_vertex->reserve(); + } +}; + +class task_group_base : no_copy { +protected: + d1::wait_context_vertex m_wait_vertex; + d1::task_group_context m_context; + + template + task_group_status internal_run_and_wait(const F& f) { + function_stack_task t{ f, r1::get_thread_reference_vertex(&m_wait_vertex) }; + + bool cancellation_status = false; + try_call([&] { + execute_and_wait(t, context(), m_wait_vertex.get_context(), context()); + }).on_completion([&] { + // TODO: the reset method is not thread-safe. Ensure the correct behavior. + cancellation_status = context().is_group_execution_cancelled(); + context().reset(); + }); + return cancellation_status ? canceled : complete; + } + + task_group_status internal_run_and_wait(d2::task_handle&& h) { + __TBB_ASSERT(h != nullptr, "Attempt to schedule empty task_handle"); + + using acs = d2::task_handle_accessor; + __TBB_ASSERT(&acs::ctx_of(h) == &context(), "Attempt to schedule task_handle into different task_group"); + + bool cancellation_status = false; + try_call([&] { + task_handle_task* task_ptr = acs::release(h); +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + // If the task has dependencies and the task_handle is not the last dependency + if (task_ptr->has_dependencies() && !task_ptr->release_dependency()) { + d1::wait(m_wait_vertex.get_context(), context()); + } else +#endif + { + execute_and_wait(*task_ptr, context(), m_wait_vertex.get_context(), context()); + } + }).on_completion([&] { + // TODO: the reset method is not thread-safe. Ensure the correct behavior. + cancellation_status = context().is_group_execution_cancelled(); + context().reset(); + }); + return cancellation_status ? canceled : complete; + } + +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + task_group_status internal_run_and_wait_for_task(d2::task_handle&& h) { + __TBB_ASSERT(h != nullptr, "Attempt to schedule empty task_handle"); + __TBB_ASSERT(&d2::task_handle_accessor::ctx_of(h) == &context(), "Attempt to schedule task_handle into different task_group"); + + task_handle_task* task_ptr = task_handle_accessor::release(h); + task_dynamic_state* state = task_ptr->get_dynamic_state(); + task_group_status status = task_group_status::not_complete; + + if (task_ptr->has_dependencies() && !task_ptr->release_dependency()) { + status = state->wait_for_completion(context()); + } else { + status = state->run_self_and_wait_for_completion(context()); + } + return status; + } +#endif + + template + d1::task* prepare_task(F&& f) { + d1::small_object_allocator alloc{}; + return alloc.new_object::type>>(std::forward(f), + r1::get_thread_reference_vertex(&m_wait_vertex), context(), alloc); + } + + d1::task_group_context& context() noexcept { + return m_context.actual_context(); + } + + template + d2::task_handle prepare_task_handle(F&& f) { + d1::small_object_allocator alloc{}; + using function_task_t = d2::function_task::type>; + d2::task_handle_task* function_task_p = alloc.new_object(std::forward(f), + r1::get_thread_reference_vertex(&m_wait_vertex), context(), alloc); + + return d2::task_handle_accessor::construct(function_task_p); + } + +public: + task_group_base(uintptr_t traits = 0) + : m_wait_vertex(0) + , m_context(d1::task_group_context::bound, d1::task_group_context::default_traits | traits) + {} + + task_group_base(d1::task_group_context& ctx) + : m_wait_vertex(0) + , m_context(&ctx) + {} + + ~task_group_base() noexcept(false) { + if (m_wait_vertex.continue_execution()) { +#if __TBB_CPP17_UNCAUGHT_EXCEPTIONS_PRESENT + bool stack_unwinding_in_progress = std::uncaught_exceptions() > 0; +#else + bool stack_unwinding_in_progress = std::uncaught_exception(); +#endif + // Always attempt to do proper cleanup to avoid inevitable memory corruption + // in case of missing wait (for the sake of better testability & debuggability) + if (!context().is_group_execution_cancelled()) + cancel(); + d1::wait(m_wait_vertex.get_context(), context()); + if (!stack_unwinding_in_progress) + throw_exception(exception_id::missing_wait); + } + } + + task_group_status wait() { + bool cancellation_status = false; + try_call([&] { + d1::wait(m_wait_vertex.get_context(), context()); + }).on_completion([&] { + // TODO: the reset method is not thread-safe. Ensure the correct behavior. + cancellation_status = m_context.is_group_execution_cancelled(); + context().reset(); + }); + return cancellation_status ? canceled : complete; + } + +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + task_group_status wait_for_task(task_completion_handle& comp_handle) { + __TBB_ASSERT(comp_handle, "Attempt to wait for completion of empty handle"); + task_dynamic_state* state = task_completion_handle_accessor::get_task_dynamic_state(comp_handle); + return state->wait_for_completion(context()); + } +#endif + + void cancel() { + context().cancel_group_execution(); + } +}; // class task_group_base + +class task_group : public task_group_base { +public: + task_group() : task_group_base(d1::task_group_context::concurrent_wait) {} + task_group(d1::task_group_context& ctx) : task_group_base(ctx) {} + + template + void run(F&& f) { + d1::spawn(*prepare_task(std::forward(f)), context()); + } + + void run(d2::task_handle&& h) { + __TBB_ASSERT(h != nullptr, "Attempt to schedule empty task_handle"); + + using acs = d2::task_handle_accessor; + __TBB_ASSERT(&acs::ctx_of(h) == &context(), "Attempt to schedule task_handle into different task_group"); + + task_handle_task* task_ptr = acs::release(h); +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + // If the task has dependencies and the task_handle is not the last dependency + if (task_ptr->has_dependencies() && !task_ptr->release_dependency()) { + return; + } +#endif + d1::spawn(*task_ptr, context()); + } + + template + d2::task_handle defer(F&& f) { + return prepare_task_handle(std::forward(f)); + + } + + template + task_group_status run_and_wait(const F& f) { + return internal_run_and_wait(f); + } + + task_group_status run_and_wait(d2::task_handle&& h) { + return internal_run_and_wait(std::move(h)); + } + +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS + task_group_status run_and_wait_for_task(d2::task_handle&& h) { + return internal_run_and_wait_for_task(std::move(h)); + } + + task_group_status get_status_of(task_completion_handle& comp_handle) { + __TBB_ASSERT(comp_handle, "Cannot get status of an empty task_completion_handle"); + task_dynamic_state* state = task_completion_handle_accessor::get_task_dynamic_state(comp_handle); + return state->get_task_status(); + } + + static void set_task_order(d2::task_handle& pred, d2::task_handle& succ) { + __TBB_ASSERT(pred != nullptr, "empty predecessor handle is not allowed for set_task_order"); + __TBB_ASSERT(succ != nullptr, "empty successor handle is not allowed for set_task_order"); + task_dynamic_state* pred_state = task_handle_accessor::get_task_dynamic_state(pred); + pred_state->add_successor(succ); + } + + static void set_task_order(d2::task_completion_handle& pred, d2::task_handle& succ) { + __TBB_ASSERT(pred != nullptr, "empty predecessor completion_handle is not allowed for set_task_order"); + __TBB_ASSERT(succ != nullptr, "empty successor handle is not allowed for set_task_order"); + task_dynamic_state* pred_state = task_completion_handle_accessor::get_task_dynamic_state(pred); + pred_state->add_successor(succ); + } + + static void transfer_this_task_completion_to(d2::task_handle& new_task) { + d1::task* curr_task = d1::current_task_ptr(); + __TBB_ASSERT(curr_task != nullptr, "transfer_this_task_completion_to was called outside of task body"); +#if __TBB_USE_OPTIONAL_RTTI && TBB_USE_DEBUG + __TBB_ASSERT(dynamic_cast(curr_task) != nullptr, + "transfer_this_task_completion_to was called from a task outside a task_group"); +#endif + static_cast(curr_task)->transfer_completion_to(new_task); + } +#endif +}; // class task_group + +class wait_delegate : public d1::delegate_base { + bool operator()() const override { + status = tg.wait(); + return true; + } +protected: + task_group& tg; + task_group_status& status; +public: + wait_delegate(task_group& a_group, task_group_status& tgs) + : tg(a_group), status(tgs) {} +}; + +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS +class wait_completion_delegate : public d1::delegate_base { + bool operator()() const override { + task_dynamic_state* state = task_completion_handle_accessor::get_task_dynamic_state(comp_handle); + d1::task_group_context& ctx = state->get_task()->ctx(); + status = state->wait_for_completion(ctx); + return true; + } +protected: + task_completion_handle& comp_handle; + task_group_status& status; +public: + wait_completion_delegate(task_completion_handle& handle, task_group_status& ts) + : comp_handle(handle), status(ts) {} +}; +#endif + +#if TBB_PREVIEW_ISOLATED_TASK_GROUP +class spawn_delegate : public d1::delegate_base { + d1::task* task_to_spawn; + d1::task_group_context& context; + bool operator()() const override { + spawn(*task_to_spawn, context); + return true; + } +public: + spawn_delegate(d1::task* a_task, d1::task_group_context& ctx) + : task_to_spawn(a_task), context(ctx) + {} +}; + +template +class run_wait_delegate : public wait_delegate { + F& func; + bool operator()() const override { + status = tg.run_and_wait(func); + return true; + } +public: + run_wait_delegate(task_group& a_group, F& a_func, task_group_status& tgs) + : wait_delegate(a_group, tgs), func(a_func) {} +}; + +class isolated_task_group : public task_group { + intptr_t this_isolation() { + return reinterpret_cast(this); + } +public: + isolated_task_group() : task_group() {} + + isolated_task_group(d1::task_group_context& ctx) : task_group(ctx) {} + + template + void run(F&& f) { + spawn_delegate sd(prepare_task(std::forward(f)), context()); + r1::isolate_within_arena(sd, this_isolation()); + } + + void run(d2::task_handle&& h) { + __TBB_ASSERT(h != nullptr, "Attempt to schedule empty task_handle"); + + using acs = d2::task_handle_accessor; + __TBB_ASSERT(&acs::ctx_of(h) == &context(), "Attempt to schedule task_handle into different task_group"); + + spawn_delegate sd(acs::release(h), context()); + r1::isolate_within_arena(sd, this_isolation()); + } + + template + task_group_status run_and_wait( const F& f ) { + task_group_status result = not_complete; + run_wait_delegate rwd(*this, f, result); + r1::isolate_within_arena(rwd, this_isolation()); + __TBB_ASSERT(result != not_complete, "premature exit from wait?"); + return result; + } + + task_group_status wait() { + task_group_status result = not_complete; + wait_delegate wd(*this, result); + r1::isolate_within_arena(wd, this_isolation()); + __TBB_ASSERT(result != not_complete, "premature exit from wait?"); + return result; + } +}; // class isolated_task_group +#endif // TBB_PREVIEW_ISOLATED_TASK_GROUP +} // namespace d2 +} // namespace detail + +inline namespace v1 { +using detail::d1::task_group_context; +using detail::d2::task_group; +#if TBB_PREVIEW_ISOLATED_TASK_GROUP +using detail::d2::isolated_task_group; +#endif + +using detail::d2::task_group_status; +using detail::d2::not_complete; +using detail::d2::complete; +using detail::d2::canceled; + +using detail::d1::is_current_task_group_canceling; +using detail::r1::missing_wait; + +using detail::d2::task_handle; +#if __TBB_PREVIEW_TASK_GROUP_EXTENSIONS +using detail::d2::task_completion_handle; +using detail::d2::task_complete; +#endif +} + +} // namespace tbb + +#if _MSC_VER && !defined(__INTEL_COMPILER) + #pragma warning(pop) // 4324 warning +#endif + +#endif // __TBB_task_group_H diff --git a/src/3rd party/oneapi/tbb/task_scheduler_observer.h b/src/3rd party/oneapi/tbb/task_scheduler_observer.h new file mode 100644 index 0000000000..a228cfe1b9 --- /dev/null +++ b/src/3rd party/oneapi/tbb/task_scheduler_observer.h @@ -0,0 +1,116 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_task_scheduler_observer_H +#define __TBB_task_scheduler_observer_H + +#include "detail/_namespace_injection.h" +#include "task_arena.h" +#include + +namespace tbb { +namespace detail { + +namespace d1 { +class task_scheduler_observer; +} + +namespace r1 { +class observer_proxy; +class observer_list; + +//! Enable or disable observation +/** For local observers the method can be used only when the current thread +has the task scheduler initialized or is attached to an arena. +Repeated calls with the same state are no-ops. **/ +TBB_EXPORT void __TBB_EXPORTED_FUNC observe(d1::task_scheduler_observer&, bool state = true); +} + +namespace d1 { +class task_scheduler_observer { + friend class r1::observer_proxy; + friend class r1::observer_list; + friend void r1::observe(d1::task_scheduler_observer&, bool); + + //! Pointer to the proxy holding this observer. + /** Observers are proxied by the scheduler to maintain persistent lists of them. **/ + std::atomic my_proxy{ nullptr }; + + //! Counter preventing the observer from being destroyed while in use by the scheduler. + /** Valid only when observation is on. **/ + std::atomic my_busy_count{ 0 }; + + //! Contains task_arena pointer + task_arena* my_task_arena{ nullptr }; +public: + //! Returns true if observation is enabled, false otherwise. + bool is_observing() const { return my_proxy.load(std::memory_order_relaxed) != nullptr; } + + //! Entry notification + /** Invoked from inside observe(true) call and whenever a worker enters the arena + this observer is associated with. If a thread is already in the arena when + the observer is activated, the entry notification is called before it + executes the first stolen task. **/ + virtual void on_scheduler_entry( bool /*is_worker*/ ) {} + + //! Exit notification + /** Invoked from inside observe(false) call and whenever a worker leaves the + arena this observer is associated with. **/ + virtual void on_scheduler_exit( bool /*is_worker*/ ) {} + + //! Construct local or global observer in inactive state (observation disabled). + /** For a local observer entry/exit notifications are invoked whenever a worker + thread joins/leaves the arena of the observer's owner thread. If a thread is + already in the arena when the observer is activated, the entry notification is + called before it executes the first stolen task. **/ + explicit task_scheduler_observer() = default; + + //! Construct local observer for a given arena in inactive state (observation disabled). + /** entry/exit notifications are invoked whenever a thread joins/leaves arena. + If a thread is already in the arena when the observer is activated, the entry notification + is called before it executes the first stolen task. **/ + explicit task_scheduler_observer(task_arena& a) : my_task_arena(&a) {} + + /** Destructor protects instance of the observer from concurrent notification. + It is recommended to disable observation before destructor of a derived class starts, + otherwise it can lead to concurrent notification callback on partly destroyed object **/ + virtual ~task_scheduler_observer() { + if (my_proxy.load(std::memory_order_acquire)) { + observe(false); + } + } + + //! Enable or disable observation + /** Warning: concurrent invocations of this method are not safe. + Repeated calls with the same state are no-ops. **/ + void observe(bool state = true) { + if( state && !my_proxy.load(std::memory_order_relaxed) ) { + __TBB_ASSERT( my_busy_count.load(std::memory_order_relaxed) == 0, "Inconsistent state of task_scheduler_observer instance"); + } + r1::observe(*this, state); + } +}; + +} // namespace d1 +} // namespace detail + +inline namespace v1 { + using detail::d1::task_scheduler_observer; +} +} // namespace tbb + + +#endif /* __TBB_task_scheduler_observer_H */ diff --git a/src/3rd party/oneapi/tbb/tbb_allocator.h b/src/3rd party/oneapi/tbb/tbb_allocator.h new file mode 100644 index 0000000000..5394b81881 --- /dev/null +++ b/src/3rd party/oneapi/tbb/tbb_allocator.h @@ -0,0 +1,126 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_tbb_allocator_H +#define __TBB_tbb_allocator_H + +#include "oneapi/tbb/detail/_utils.h" +#include "detail/_namespace_injection.h" +#include +#include + +#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT +#include +#endif + +namespace tbb { +namespace detail { + +namespace r1 { +TBB_EXPORT void* __TBB_EXPORTED_FUNC allocate_memory(std::size_t size); +TBB_EXPORT void __TBB_EXPORTED_FUNC deallocate_memory(void* p); +TBB_EXPORT bool __TBB_EXPORTED_FUNC is_tbbmalloc_used(); +} + +namespace d1 { + +template +class tbb_allocator { +public: + using value_type = T; + using propagate_on_container_move_assignment = std::true_type; + + //! Always defined for TBB containers (supported since C++17 for std containers) + using is_always_equal = std::true_type; + + //! Specifies current allocator + enum malloc_type { + scalable, + standard + }; + + tbb_allocator() = default; + template tbb_allocator(const tbb_allocator&) noexcept {} + + //! Allocate space for n objects. + __TBB_nodiscard T* allocate(std::size_t n) { + return static_cast(r1::allocate_memory(n * sizeof(value_type))); + } + + //! Free previously allocated block of memory. + void deallocate(T* p, std::size_t) { + r1::deallocate_memory(p); + } + + //! Returns current allocator + static malloc_type allocator_type() { + return r1::is_tbbmalloc_used() ? standard : scalable; + } + +#if TBB_ALLOCATOR_TRAITS_BROKEN + using pointer = value_type*; + using const_pointer = const value_type*; + using reference = value_type&; + using const_reference = const value_type&; + using difference_type = std::ptrdiff_t; + using size_type = std::size_t; + template struct rebind { + using other = tbb_allocator; + }; + //! Largest value for which method allocate might succeed. + size_type max_size() const noexcept { + size_type max = ~(std::size_t(0)) / sizeof(value_type); + return (max > 0 ? max : 1); + } + template + void construct(U *p, Args&&... args) + { ::new (p) U(std::forward(args)...); } + void destroy( pointer p ) { p->~value_type(); } + pointer address(reference x) const { return &x; } + const_pointer address(const_reference x) const { return &x; } +#endif // TBB_ALLOCATOR_TRAITS_BROKEN +}; + +#if TBB_ALLOCATOR_TRAITS_BROKEN + template<> + class tbb_allocator { + public: + using pointer = void*; + using const_pointer = const void*; + using value_type = void; + template struct rebind { + using other = tbb_allocator; + }; + }; +#endif + +template +inline bool operator==(const tbb_allocator&, const tbb_allocator&) noexcept { return true; } + +#if !__TBB_CPP20_COMPARISONS_PRESENT +template +inline bool operator!=(const tbb_allocator&, const tbb_allocator&) noexcept { return false; } +#endif + +} // namespace d1 +} // namespace detail + +inline namespace v1 { +using detail::d1::tbb_allocator; +} // namespace v1 +} // namespace tbb + +#endif /* __TBB_tbb_allocator_H */ diff --git a/src/3rd party/oneapi/tbb/tbbmalloc_proxy.h b/src/3rd party/oneapi/tbb/tbbmalloc_proxy.h new file mode 100644 index 0000000000..0ba38f215e --- /dev/null +++ b/src/3rd party/oneapi/tbb/tbbmalloc_proxy.h @@ -0,0 +1,65 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* +Replacing the standard memory allocation routines in Microsoft* C/C++ RTL +(malloc/free, global new/delete, etc.) with the TBB memory allocator. + +Include the following header to a source of any binary which is loaded during +application startup + +#include "oneapi/tbb/tbbmalloc_proxy.h" + +or add following parameters to the linker options for the binary which is +loaded during application startup. It can be either exe-file or dll. + +For win32 +tbbmalloc_proxy.lib /INCLUDE:"___TBB_malloc_proxy" +win64 +tbbmalloc_proxy.lib /INCLUDE:"__TBB_malloc_proxy" +*/ + +#ifndef __TBB_tbbmalloc_proxy_H +#define __TBB_tbbmalloc_proxy_H + +#if _MSC_VER + +#ifdef _DEBUG + #pragma comment(lib, "tbbmalloc_proxy_debug.lib") +#else + #pragma comment(lib, "tbbmalloc_proxy.lib") +#endif + +#if defined(_WIN64) + #pragma comment(linker, "/include:__TBB_malloc_proxy") +#else + #pragma comment(linker, "/include:___TBB_malloc_proxy") +#endif + +#else +/* Primarily to support MinGW */ + +extern "C" void __TBB_malloc_proxy(); +struct __TBB_malloc_proxy_caller { + __TBB_malloc_proxy_caller() { __TBB_malloc_proxy(); } +} volatile __TBB_malloc_proxy_helper_object; + +#endif // _MSC_VER + +/* Public Windows API */ +extern "C" int TBB_malloc_replacement_log(char *** function_replacement_log_ptr); + +#endif //__TBB_tbbmalloc_proxy_H diff --git a/src/3rd party/oneapi/tbb/tick_count.h b/src/3rd party/oneapi/tbb/tick_count.h new file mode 100644 index 0000000000..2caa56ba18 --- /dev/null +++ b/src/3rd party/oneapi/tbb/tick_count.h @@ -0,0 +1,99 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_tick_count_H +#define __TBB_tick_count_H + +#include + +#include "detail/_namespace_injection.h" + +namespace tbb { +namespace detail { +namespace d1 { + + +//! Absolute timestamp +/** @ingroup timing */ +class tick_count { +public: + using clock_type = typename std::conditional::type; + + //! Relative time interval. + class interval_t : public clock_type::duration { + public: + //! Construct a time interval representing zero time duration + interval_t() : clock_type::duration(clock_type::duration::zero()) {} + + //! Construct a time interval representing sec seconds time duration + explicit interval_t( double sec ) + : clock_type::duration(std::chrono::duration_cast(std::chrono::duration(sec))) {} + + //! Return the length of a time interval in seconds + double seconds() const { + return std::chrono::duration_cast>(*this).count(); + } + + //! Extract the intervals from the tick_counts and subtract them. + friend interval_t operator-( const tick_count& t1, const tick_count& t0 ); + + //! Add two intervals. + friend interval_t operator+( const interval_t& i, const interval_t& j ) { + return interval_t(std::chrono::operator+(i, j)); + } + + //! Subtract two intervals. + friend interval_t operator-( const interval_t& i, const interval_t& j ) { + return interval_t(std::chrono::operator-(i, j)); + } + + private: + explicit interval_t( clock_type::duration value_ ) : clock_type::duration(value_) {} + }; + + tick_count() = default; + + //! Return current time. + static tick_count now() { + return clock_type::now(); + } + + //! Subtract two timestamps to get the time interval between + friend interval_t operator-( const tick_count& t1, const tick_count& t0 ) { + return tick_count::interval_t(t1.my_time_point - t0.my_time_point); + } + + //! Return the resolution of the clock in seconds per tick. + static double resolution() { + return static_cast(interval_t::period::num) / interval_t::period::den; + } + +private: + clock_type::time_point my_time_point; + tick_count( clock_type::time_point tp ) : my_time_point(tp) {} +}; + +} // namespace d1 +} // namespace detail + +inline namespace v1 { + using detail::d1::tick_count; +} // namespace v1 + +} // namespace tbb + +#endif /* __TBB_tick_count_H */ diff --git a/src/3rd party/oneapi/tbb/version.h b/src/3rd party/oneapi/tbb/version.h new file mode 100644 index 0000000000..a7887dfb31 --- /dev/null +++ b/src/3rd party/oneapi/tbb/version.h @@ -0,0 +1,119 @@ +/* + Copyright (c) 2005-2025 Intel Corporation + Copyright (c) 2025 UXL Foundation Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_version_H +#define __TBB_version_H + +// Exclude all includes during .rc files compilation +#ifndef RC_INVOKED + #include "detail/_config.h" + #include "detail/_namespace_injection.h" +#else + #define __TBB_STRING_AUX(x) #x + #define __TBB_STRING(x) __TBB_STRING_AUX(x) +#endif + +// Product version +#define TBB_VERSION_MAJOR 2023 +// Update version +#define TBB_VERSION_MINOR 0 +// "Patch" version for custom releases +#define TBB_VERSION_PATCH 0 +// Suffix string +#define __TBB_VERSION_SUFFIX "" +// Full official version string +#define TBB_VERSION_STRING \ + __TBB_STRING(TBB_VERSION_MAJOR) "." \ + __TBB_STRING(TBB_VERSION_MINOR) "." \ + __TBB_STRING(TBB_VERSION_PATCH) \ + __TBB_VERSION_SUFFIX + +// OneAPI oneTBB specification version +#define ONETBB_SPEC_VERSION 104 +// Full interface version +#define TBB_INTERFACE_VERSION 12180 +// Major interface version +#define TBB_INTERFACE_VERSION_MAJOR (TBB_INTERFACE_VERSION/1000) +// Minor interface version +#define TBB_INTERFACE_VERSION_MINOR (TBB_INTERFACE_VERSION%1000/10) + +// The binary compatibility version +// To be used in SONAME, manifests, etc. +#define __TBB_BINARY_VERSION 12 + +//! TBB_VERSION support +#ifndef TBB_ENDL +#define TBB_ENDL "\n" +#endif + +//TBB_REVAMP_TODO: consider enabling version_string.ver generation +//TBB_REVAMP_TODO: #include "version_string.ver" + +#define __TBB_ONETBB_SPEC_VERSION(N) #N ": SPECIFICATION VERSION\t" __TBB_STRING(ONETBB_SPEC_VERSION) TBB_ENDL +#define __TBB_VERSION_NUMBER(N) #N ": VERSION\t\t" TBB_VERSION_STRING TBB_ENDL +#define __TBB_INTERFACE_VERSION_NUMBER(N) #N ": INTERFACE VERSION\t" __TBB_STRING(TBB_INTERFACE_VERSION) TBB_ENDL + +#ifndef TBB_USE_DEBUG + #define __TBB_VERSION_USE_DEBUG(N) #N ": TBB_USE_DEBUG\tundefined" TBB_ENDL +#elif TBB_USE_DEBUG==0 + #define __TBB_VERSION_USE_DEBUG(N) #N ": TBB_USE_DEBUG\t0" TBB_ENDL +#elif TBB_USE_DEBUG==1 + #define __TBB_VERSION_USE_DEBUG(N) #N ": TBB_USE_DEBUG\t1" TBB_ENDL +#elif TBB_USE_DEBUG==2 + #define __TBB_VERSION_USE_DEBUG(N) #N ": TBB_USE_DEBUG\t2" TBB_ENDL +#else + #error Unexpected value for TBB_USE_DEBUG +#endif + +#ifndef TBB_USE_ASSERT + #define __TBB_VERSION_USE_ASSERT(N) #N ": TBB_USE_ASSERT\tundefined" TBB_ENDL +#elif TBB_USE_ASSERT==0 + #define __TBB_VERSION_USE_ASSERT(N) #N ": TBB_USE_ASSERT\t0" TBB_ENDL +#elif TBB_USE_ASSERT==1 + #define __TBB_VERSION_USE_ASSERT(N) #N ": TBB_USE_ASSERT\t1" TBB_ENDL +#elif TBB_USE_ASSERT==2 + #define __TBB_VERSION_USE_ASSERT(N) #N ": TBB_USE_ASSERT\t2" TBB_ENDL +#else + #error Unexpected value for TBB_USE_ASSERT +#endif + +#define TBB_VERSION_STRINGS_P(N) \ + __TBB_ONETBB_SPEC_VERSION(N) \ + __TBB_VERSION_NUMBER(N) \ + __TBB_INTERFACE_VERSION_NUMBER(N) \ + __TBB_VERSION_USE_DEBUG(N) \ + __TBB_VERSION_USE_ASSERT(N) + +#define TBB_VERSION_STRINGS TBB_VERSION_STRINGS_P(oneTBB) +#define TBBMALLOC_VERSION_STRINGS TBB_VERSION_STRINGS_P(TBBmalloc) + +//! The function returns the version string for the Intel(R) oneAPI Threading Building Blocks (oneTBB) +//! shared library being used. +/** + * The returned pointer is an address of a string in the shared library. + * It can be different than the TBB_VERSION_STRING obtained at compile time. + */ +extern "C" TBB_EXPORT const char* __TBB_EXPORTED_FUNC TBB_runtime_version(); + +//! The function returns the interface version of the oneTBB shared library being used. +/** + * The returned version is determined at runtime, not at compile/link time. + * It can be different than the value of TBB_INTERFACE_VERSION obtained at compile time. + */ +extern "C" TBB_EXPORT int __TBB_EXPORTED_FUNC TBB_runtime_interface_version(); + +#endif // __TBB_version_H diff --git a/src/3rd party/serial/tbb/parallel_for.h b/src/3rd party/serial/tbb/parallel_for.h deleted file mode 100644 index cfe6ccf639..0000000000 --- a/src/3rd party/serial/tbb/parallel_for.h +++ /dev/null @@ -1,226 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include "../../tbb/internal/_deprecated_header_message_guard.h" - -#if !defined(__TBB_show_deprecation_message_parallel_for_H) && defined(__TBB_show_deprecated_header_message) -#define __TBB_show_deprecation_message_parallel_for_H -#pragma message("TBB Warning: serial/tbb/parallel_for.h is deprecated. For details, please see Deprecated Features appendix in the TBB reference manual.") -#endif - -#if defined(__TBB_show_deprecated_header_message) -#undef __TBB_show_deprecated_header_message -#endif - -#ifndef __TBB_SERIAL_parallel_for_H -#define __TBB_SERIAL_parallel_for_H - -#include "tbb_annotate.h" - -#ifndef __TBB_NORMAL_EXECUTION -#include "tbb/blocked_range.h" -#include "tbb/partitioner.h" -#endif - -#if TBB_USE_EXCEPTIONS -#include -#include // required to construct std exception classes -#else -#include -#include -#endif - -namespace tbb { -namespace serial { -namespace interface9 { - -// parallel_for serial annotated implementation - -template< typename Range, typename Body, typename Partitioner > -class start_for : tbb::internal::no_copy { - Range my_range; - const Body my_body; - typename Partitioner::task_partition_type my_partition; - void execute(); - - //! Constructor for root task. - start_for( const Range& range, const Body& body, Partitioner& partitioner ) : - my_range( range ), - my_body( body ), - my_partition( partitioner ) - { - } - - //! Splitting constructor used to generate children. - /** this becomes left child. Newly constructed object is right child. */ - start_for( start_for& parent_, typename Partitioner::split_type& split_obj ) : - my_range( parent_.my_range, split_obj ), - my_body( parent_.my_body ), - my_partition( parent_.my_partition, split_obj ) - { - } - -public: - static void run( const Range& range, const Body& body, Partitioner& partitioner ) { - if( !range.empty() ) { - ANNOTATE_SITE_BEGIN( tbb_parallel_for ); - { - start_for a( range, body, partitioner ); - a.execute(); - } - ANNOTATE_SITE_END( tbb_parallel_for ); - } - } -}; - -template< typename Range, typename Body, typename Partitioner > -void start_for< Range, Body, Partitioner >::execute() { - if( !my_range.is_divisible() || !my_partition.is_divisible() ) { - ANNOTATE_TASK_BEGIN( tbb_parallel_for_range ); - { - my_body( my_range ); - } - ANNOTATE_TASK_END( tbb_parallel_for_range ); - } else { - typename Partitioner::split_type split_obj; - start_for b( *this, split_obj ); - this->execute(); // Execute the left interval first to keep the serial order. - b.execute(); // Execute the right interval then. - } -} - -//! Parallel iteration over range with default partitioner. -/** @ingroup algorithms **/ -template -__TBB_DEPRECATED_VERBOSE void parallel_for( const Range& range, const Body& body ) { - serial::interface9::start_for::run(range,body,__TBB_DEFAULT_PARTITIONER()); -} - -//! Parallel iteration over range with simple partitioner. -/** @ingroup algorithms **/ -template -__TBB_DEPRECATED_VERBOSE void parallel_for( const Range& range, const Body& body, const simple_partitioner& partitioner ) { - serial::interface9::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with auto_partitioner. -/** @ingroup algorithms **/ -template -__TBB_DEPRECATED_VERBOSE void parallel_for( const Range& range, const Body& body, const auto_partitioner& partitioner ) { - serial::interface9::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with static_partitioner. -/** @ingroup algorithms **/ -template -__TBB_DEPRECATED_VERBOSE void parallel_for( const Range& range, const Body& body, const static_partitioner& partitioner ) { - serial::interface9::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with affinity_partitioner. -/** @ingroup algorithms **/ -template -__TBB_DEPRECATED_VERBOSE void parallel_for( const Range& range, const Body& body, affinity_partitioner& partitioner ) { - serial::interface9::start_for::run(range,body,partitioner); -} - -//! Implementation of parallel iteration over stepped range of integers with explicit step and partitioner (ignored) -template -void parallel_for_impl(Index first, Index last, Index step, const Function& f, Partitioner& ) { - if (step <= 0 ) { -#if TBB_USE_EXCEPTIONS - throw std::invalid_argument( "nonpositive_step" ); -#else - std::cerr << "nonpositive step in a call to parallel_for" << std::endl; - std::abort(); -#endif - } else if (last > first) { - // Above "else" avoids "potential divide by zero" warning on some platforms - ANNOTATE_SITE_BEGIN( tbb_parallel_for ); - for( Index i = first; i < last; i = i + step ) { - ANNOTATE_TASK_BEGIN( tbb_parallel_for_iteration ); - { f( i ); } - ANNOTATE_TASK_END( tbb_parallel_for_iteration ); - } - ANNOTATE_SITE_END( tbb_parallel_for ); - } -} - -//! Parallel iteration over a range of integers with explicit step and default partitioner -template -__TBB_DEPRECATED_VERBOSE void parallel_for(Index first, Index last, Index step, const Function& f) { - parallel_for_impl(first, last, step, f, auto_partitioner()); -} -//! Parallel iteration over a range of integers with explicit step and simple partitioner -template -__TBB_DEPRECATED_VERBOSE void parallel_for(Index first, Index last, Index step, const Function& f, const simple_partitioner& p) { - parallel_for_impl(first, last, step, f, p); -} -//! Parallel iteration over a range of integers with explicit step and auto partitioner -template -__TBB_DEPRECATED_VERBOSE void parallel_for(Index first, Index last, Index step, const Function& f, const auto_partitioner& p) { - parallel_for_impl(first, last, step, f, p); -} -//! Parallel iteration over a range of integers with explicit step and static partitioner -template -__TBB_DEPRECATED_VERBOSE void parallel_for(Index first, Index last, Index step, const Function& f, const static_partitioner& p) { - parallel_for_impl(first, last, step, f, p); -} -//! Parallel iteration over a range of integers with explicit step and affinity partitioner -template -__TBB_DEPRECATED_VERBOSE void parallel_for(Index first, Index last, Index step, const Function& f, affinity_partitioner& p) { - parallel_for_impl(first, last, step, f, p); -} - -//! Parallel iteration over a range of integers with default step and default partitioner -template -__TBB_DEPRECATED_VERBOSE void parallel_for(Index first, Index last, const Function& f) { - parallel_for_impl(first, last, static_cast(1), f, auto_partitioner()); -} -//! Parallel iteration over a range of integers with default step and simple partitioner -template -__TBB_DEPRECATED_VERBOSE void parallel_for(Index first, Index last, const Function& f, const simple_partitioner& p) { - parallel_for_impl(first, last, static_cast(1), f, p); -} -//! Parallel iteration over a range of integers with default step and auto partitioner -template -__TBB_DEPRECATED_VERBOSE void parallel_for(Index first, Index last, const Function& f, const auto_partitioner& p) { - parallel_for_impl(first, last, static_cast(1), f, p); -} -//! Parallel iteration over a range of integers with default step and static partitioner -template -__TBB_DEPRECATED_VERBOSE void parallel_for(Index first, Index last, const Function& f, const static_partitioner& p) { - parallel_for_impl(first, last, static_cast(1), f, p); -} -//! Parallel iteration over a range of integers with default step and affinity_partitioner -template -__TBB_DEPRECATED_VERBOSE void parallel_for(Index first, Index last, const Function& f, affinity_partitioner& p) { - parallel_for_impl(first, last, static_cast(1), f, p); -} - -} // namespace interfaceX - -using interface9::parallel_for; - -} // namespace serial - -#ifndef __TBB_NORMAL_EXECUTION -using serial::interface9::parallel_for; -#endif - -} // namespace tbb - -#endif /* __TBB_SERIAL_parallel_for_H */ diff --git a/src/3rd party/tbb/aggregator.h b/src/3rd party/tbb/aggregator.h deleted file mode 100644 index 9d42c3dc95..0000000000 --- a/src/3rd party/tbb/aggregator.h +++ /dev/null @@ -1,204 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB__aggregator_H -#define __TBB__aggregator_H - -#define __TBB_aggregator_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#if !TBB_PREVIEW_AGGREGATOR -#error Set TBB_PREVIEW_AGGREGATOR before including aggregator.h -#endif - -#include "atomic.h" -#include "tbb_profiling.h" - -namespace tbb { -namespace interface6 { - -using namespace tbb::internal; - -class aggregator_operation { - template friend class aggregator_ext; - uintptr_t status; - aggregator_operation* my_next; -public: - enum aggregator_operation_status { agg_waiting=0, agg_finished }; - aggregator_operation() : status(agg_waiting), my_next(NULL) {} - /// Call start before handling this operation - void start() { call_itt_notify(acquired, &status); } - /// Call finish when done handling this operation - /** The operation will be released to its originating thread, and possibly deleted. */ - void finish() { itt_store_word_with_release(status, uintptr_t(agg_finished)); } - aggregator_operation* next() { return itt_hide_load_word(my_next);} - void set_next(aggregator_operation* n) { itt_hide_store_word(my_next, n); } -}; - -namespace internal { - -class basic_operation_base : public aggregator_operation { - friend class basic_handler; - virtual void apply_body() = 0; -public: - basic_operation_base() : aggregator_operation() {} - virtual ~basic_operation_base() {} -}; - -template -class basic_operation : public basic_operation_base, no_assign { - const Body& my_body; - void apply_body() __TBB_override { my_body(); } -public: - basic_operation(const Body& b) : basic_operation_base(), my_body(b) {} -}; - -class basic_handler { -public: - basic_handler() {} - void operator()(aggregator_operation* op_list) const { - while (op_list) { - // ITT note: &(op_list->status) tag is used to cover accesses to the operation data. - // The executing thread "acquires" the tag (see start()) and then performs - // the associated operation w/o triggering a race condition diagnostics. - // A thread that created the operation is waiting for its status (see execute_impl()), - // so when this thread is done with the operation, it will "release" the tag - // and update the status (see finish()) to give control back to the waiting thread. - basic_operation_base& request = static_cast(*op_list); - // IMPORTANT: need to advance op_list to op_list->next() before calling request.finish() - op_list = op_list->next(); - request.start(); - request.apply_body(); - request.finish(); - } - } -}; - -} // namespace internal - -//! Aggregator base class and expert interface -/** An aggregator for collecting operations coming from multiple sources and executing - them serially on a single thread. */ -template -class aggregator_ext : tbb::internal::no_copy { -public: - aggregator_ext(const handler_type& h) : handler_busy(0), handle_operations(h) { mailbox = NULL; } - - //! EXPERT INTERFACE: Enter a user-made operation into the aggregator's mailbox. - /** Details of user-made operations must be handled by user-provided handler */ - void process(aggregator_operation *op) { execute_impl(*op); } - -protected: - /** Place operation in mailbox, then either handle mailbox or wait for the operation - to be completed by a different thread. */ - void execute_impl(aggregator_operation& op) { - aggregator_operation* res; - - // ITT note: &(op.status) tag is used to cover accesses to this operation. This - // thread has created the operation, and now releases it so that the handler - // thread may handle the associated operation w/o triggering a race condition; - // thus this tag will be acquired just before the operation is handled in the - // handle_operations functor. - call_itt_notify(releasing, &(op.status)); - // insert the operation into the list - do { - // ITT may flag the following line as a race; it is a false positive: - // This is an atomic read; we don't provide itt_hide_load_word for atomics - op.my_next = res = mailbox; // NOT A RACE - } while (mailbox.compare_and_swap(&op, res) != res); - if (!res) { // first in the list; handle the operations - // ITT note: &mailbox tag covers access to the handler_busy flag, which this - // waiting handler thread will try to set before entering handle_operations. - call_itt_notify(acquired, &mailbox); - start_handle_operations(); - __TBB_ASSERT(op.status, NULL); - } - else { // not first; wait for op to be ready - call_itt_notify(prepare, &(op.status)); - spin_wait_while_eq(op.status, uintptr_t(aggregator_operation::agg_waiting)); - itt_load_word_with_acquire(op.status); - } - } - - -private: - //! An atomically updated list (aka mailbox) of aggregator_operations - atomic mailbox; - - //! Controls thread access to handle_operations - /** Behaves as boolean flag where 0=false, 1=true */ - uintptr_t handler_busy; - - handler_type handle_operations; - - //! Trigger the handling of operations when the handler is free - void start_handle_operations() { - aggregator_operation *pending_operations; - - // ITT note: &handler_busy tag covers access to mailbox as it is passed - // between active and waiting handlers. Below, the waiting handler waits until - // the active handler releases, and the waiting handler acquires &handler_busy as - // it becomes the active_handler. The release point is at the end of this - // function, when all operations in mailbox have been handled by the - // owner of this aggregator. - call_itt_notify(prepare, &handler_busy); - // get handler_busy: only one thread can possibly spin here at a time - spin_wait_until_eq(handler_busy, uintptr_t(0)); - call_itt_notify(acquired, &handler_busy); - // acquire fence not necessary here due to causality rule and surrounding atomics - __TBB_store_with_release(handler_busy, uintptr_t(1)); - - // ITT note: &mailbox tag covers access to the handler_busy flag itself. - // Capturing the state of the mailbox signifies that handler_busy has been - // set and a new active handler will now process that list's operations. - call_itt_notify(releasing, &mailbox); - // grab pending_operations - pending_operations = mailbox.fetch_and_store(NULL); - - // handle all the operations - handle_operations(pending_operations); - - // release the handler - itt_store_word_with_release(handler_busy, uintptr_t(0)); - } -}; - -//! Basic aggregator interface -class aggregator : private aggregator_ext { -public: - aggregator() : aggregator_ext(internal::basic_handler()) {} - //! BASIC INTERFACE: Enter a function for exclusive execution by the aggregator. - /** The calling thread stores the function object in a basic_operation and - places the operation in the aggregator's mailbox */ - template - void execute(const Body& b) { - internal::basic_operation op(b); - this->execute_impl(op); - } -}; - -} // namespace interface6 - -using interface6::aggregator; -using interface6::aggregator_ext; -using interface6::aggregator_operation; - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_aggregator_H_include_area - -#endif // __TBB__aggregator_H diff --git a/src/3rd party/tbb/aligned_space.h b/src/3rd party/tbb/aligned_space.h deleted file mode 100644 index d91fd6f58c..0000000000 --- a/src/3rd party/tbb/aligned_space.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include "internal/_deprecated_header_message_guard.h" - -#if !defined(__TBB_show_deprecation_message_aligned_space_H) && defined(__TBB_show_deprecated_header_message) -#define __TBB_show_deprecation_message_aligned_space_H -#pragma message("TBB Warning: tbb/aligned_space.h is deprecated. For details, please see Deprecated Features appendix in the TBB reference manual.") -#endif - -#if defined(__TBB_show_deprecated_header_message) -#undef __TBB_show_deprecated_header_message -#endif - -#ifndef __TBB_aligned_space_H -#define __TBB_aligned_space_H - -#define __TBB_aligned_space_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "tbb_stddef.h" -#include "tbb_machine.h" - -namespace tbb { - -//! Block of space aligned sufficiently to construct an array T with N elements. -/** The elements are not constructed or destroyed by this class. - @ingroup memory_allocation */ -template -class __TBB_DEPRECATED_VERBOSE_MSG("tbb::aligned_space is deprecated, use std::aligned_storage") aligned_space { -private: - typedef __TBB_TypeWithAlignmentAtLeastAsStrict(T) element_type; - element_type array[(sizeof(T)*N+sizeof(element_type)-1)/sizeof(element_type)]; -public: - //! Pointer to beginning of array - T* begin() const {return internal::punned_cast(this);} - - //! Pointer to one past last element in array. - T* end() const {return begin()+N;} -}; - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_aligned_space_H_include_area - -#endif /* __TBB_aligned_space_H */ diff --git a/src/3rd party/tbb/atomic.h b/src/3rd party/tbb/atomic.h deleted file mode 100644 index 33c54af4e0..0000000000 --- a/src/3rd party/tbb/atomic.h +++ /dev/null @@ -1,586 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include "internal/_deprecated_header_message_guard.h" - -#if !defined(__TBB_show_deprecation_message_atomic_H) && defined(__TBB_show_deprecated_header_message) -#define __TBB_show_deprecation_message_atomic_H -#pragma message("TBB Warning: tbb/atomic.h is deprecated. For details, please see Deprecated Features appendix in the TBB reference manual.") -#endif - -#if defined(__TBB_show_deprecated_header_message) -#undef __TBB_show_deprecated_header_message -#endif - -#ifndef __TBB_atomic_H -#define __TBB_atomic_H - -#define __TBB_atomic_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include - -#if _MSC_VER -#define __TBB_LONG_LONG __int64 -#else -#define __TBB_LONG_LONG long long -#endif /* _MSC_VER */ - -#include "tbb_machine.h" - -#if _MSC_VER && !__INTEL_COMPILER - // Suppress overzealous compiler warnings till the end of the file - #pragma warning (push) - #pragma warning (disable: 4244 4267 4512) -#endif - -namespace tbb { - -//! Specifies memory semantics. -enum memory_semantics { - //! Sequential consistency - full_fence, - //! Acquire - acquire, - //! Release - release, - //! No ordering - relaxed -}; - -//! @cond INTERNAL -namespace internal { - -#if __TBB_ALIGNAS_PRESENT - #define __TBB_DECL_ATOMIC_FIELD(t,f,a) alignas(a) t f; -#elif __TBB_ATTRIBUTE_ALIGNED_PRESENT - #define __TBB_DECL_ATOMIC_FIELD(t,f,a) t f __attribute__ ((aligned(a))); -#elif __TBB_DECLSPEC_ALIGN_PRESENT - #define __TBB_DECL_ATOMIC_FIELD(t,f,a) __declspec(align(a)) t f; -#else - #error Do not know syntax for forcing alignment. -#endif - -template -struct atomic_rep; // Primary template declared, but never defined. - -template<> -struct atomic_rep<1> { // Specialization - typedef int8_t word; -}; -template<> -struct atomic_rep<2> { // Specialization - typedef int16_t word; -}; -template<> -struct atomic_rep<4> { // Specialization -#if _MSC_VER && !_WIN64 - // Work-around that avoids spurious /Wp64 warnings - typedef intptr_t word; -#else - typedef int32_t word; -#endif -}; -#if __TBB_64BIT_ATOMICS -template<> -struct atomic_rep<8> { // Specialization - typedef int64_t word; -}; -#endif - -template -struct aligned_storage; - -//the specializations are needed to please MSVC syntax of __declspec(align()) which accept _literal_ constants only -#if __TBB_ATOMIC_CTORS - #define ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(S) \ - template \ - struct aligned_storage { \ - __TBB_DECL_ATOMIC_FIELD(value_type,my_value,S) \ - aligned_storage() = default ; \ - constexpr aligned_storage(value_type value):my_value(value){} \ - }; \ - -#else - #define ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(S) \ - template \ - struct aligned_storage { \ - __TBB_DECL_ATOMIC_FIELD(value_type,my_value,S) \ - }; \ - -#endif - -template -struct aligned_storage { - value_type my_value; -#if __TBB_ATOMIC_CTORS - aligned_storage() = default ; - constexpr aligned_storage(value_type value):my_value(value){} -#endif -}; - -ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(2) -ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(4) -#if __TBB_64BIT_ATOMICS -ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(8) -#endif - -template -struct atomic_traits; // Primary template declared, but not defined. - -#define __TBB_DECL_FENCED_ATOMIC_PRIMITIVES(S,M) \ - template<> struct atomic_traits { \ - typedef atomic_rep::word word; \ - inline static word compare_and_swap( volatile void* location, word new_value, word comparand ) { \ - return __TBB_machine_cmpswp##S##M(location,new_value,comparand); \ - } \ - inline static word fetch_and_add( volatile void* location, word addend ) { \ - return __TBB_machine_fetchadd##S##M(location,addend); \ - } \ - inline static word fetch_and_store( volatile void* location, word value ) { \ - return __TBB_machine_fetchstore##S##M(location,value); \ - } \ - }; - -#define __TBB_DECL_ATOMIC_PRIMITIVES(S) \ - template \ - struct atomic_traits { \ - typedef atomic_rep::word word; \ - inline static word compare_and_swap( volatile void* location, word new_value, word comparand ) { \ - return __TBB_machine_cmpswp##S(location,new_value,comparand); \ - } \ - inline static word fetch_and_add( volatile void* location, word addend ) { \ - return __TBB_machine_fetchadd##S(location,addend); \ - } \ - inline static word fetch_and_store( volatile void* location, word value ) { \ - return __TBB_machine_fetchstore##S(location,value); \ - } \ - }; - -template -struct atomic_load_store_traits; // Primary template declaration - -#define __TBB_DECL_ATOMIC_LOAD_STORE_PRIMITIVES(M) \ - template<> struct atomic_load_store_traits { \ - template \ - inline static T load( const volatile T& location ) { \ - return __TBB_load_##M( location ); \ - } \ - template \ - inline static void store( volatile T& location, T value ) { \ - __TBB_store_##M( location, value ); \ - } \ - } - -#if __TBB_USE_FENCED_ATOMICS -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(1,full_fence) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(2,full_fence) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(4,full_fence) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(1,acquire) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(2,acquire) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(4,acquire) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(1,release) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(2,release) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(4,release) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(1,relaxed) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(2,relaxed) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(4,relaxed) -#if __TBB_64BIT_ATOMICS -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(8,full_fence) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(8,acquire) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(8,release) -__TBB_DECL_FENCED_ATOMIC_PRIMITIVES(8,relaxed) -#endif -#else /* !__TBB_USE_FENCED_ATOMICS */ -__TBB_DECL_ATOMIC_PRIMITIVES(1) -__TBB_DECL_ATOMIC_PRIMITIVES(2) -__TBB_DECL_ATOMIC_PRIMITIVES(4) -#if __TBB_64BIT_ATOMICS -__TBB_DECL_ATOMIC_PRIMITIVES(8) -#endif -#endif /* !__TBB_USE_FENCED_ATOMICS */ - -__TBB_DECL_ATOMIC_LOAD_STORE_PRIMITIVES(full_fence); -__TBB_DECL_ATOMIC_LOAD_STORE_PRIMITIVES(acquire); -__TBB_DECL_ATOMIC_LOAD_STORE_PRIMITIVES(release); -__TBB_DECL_ATOMIC_LOAD_STORE_PRIMITIVES(relaxed); - -//! Additive inverse of 1 for type T. -/** Various compilers issue various warnings if -1 is used with various integer types. - The baroque expression below avoids all the warnings (we hope). */ -#define __TBB_MINUS_ONE(T) (T(T(0)-T(1))) - -//! Base class that provides basic functionality for atomic without fetch_and_add. -/** Works for any type T that has the same size as an integral type, has a trivial constructor/destructor, - and can be copied/compared by memcpy/memcmp. */ -template -struct atomic_impl { -protected: - aligned_storage my_storage; -private: - //TODO: rechecks on recent versions of gcc if union is still the _only_ way to do a conversion without warnings - //! Union type used to convert type T to underlying integral type. - template - union converter { - typedef typename atomic_rep::word bits_type; - converter(){} - converter(value_type a_value) : value(a_value) {} - value_type value; - bits_type bits; - }; - - template - static typename converter::bits_type to_bits(value_t value){ - return converter(value).bits; - } - template - static value_t to_value(typename converter::bits_type bits){ - converter u; - u.bits = bits; - return u.value; - } - - template - union ptr_converter; //Primary template declared, but never defined. - - template - union ptr_converter { - ptr_converter(){} - ptr_converter(value_t* a_value) : value(a_value) {} - value_t* value; - uintptr_t bits; - }; - //TODO: check if making to_bits accepting reference (thus unifying it with to_bits_ref) - //does not hurt performance - template - static typename converter::bits_type & to_bits_ref(value_t& value){ - //TODO: this #ifdef is temporary workaround, as union conversion seems to fail - //on suncc for 64 bit types for 32 bit target - #if !__SUNPRO_CC - return *(typename converter::bits_type*)ptr_converter(&value).bits; - #else - return *(typename converter::bits_type*)(&value); - #endif - } - - -public: - typedef T value_type; - -#if __TBB_ATOMIC_CTORS - atomic_impl() = default ; - constexpr atomic_impl(value_type value):my_storage(value){} -#endif - template - value_type fetch_and_store( value_type value ) { - return to_value( - internal::atomic_traits::fetch_and_store( &my_storage.my_value, to_bits(value) ) - ); - } - - value_type fetch_and_store( value_type value ) { - return fetch_and_store(value); - } - - template - value_type compare_and_swap( value_type value, value_type comparand ) { - return to_value( - internal::atomic_traits::compare_and_swap( &my_storage.my_value, to_bits(value), to_bits(comparand) ) - ); - } - - value_type compare_and_swap( value_type value, value_type comparand ) { - return compare_and_swap(value,comparand); - } - - operator value_type() const volatile { // volatile qualifier here for backwards compatibility - return to_value( - __TBB_load_with_acquire( to_bits_ref(my_storage.my_value) ) - ); - } - - template - value_type load () const { - return to_value( - internal::atomic_load_store_traits::load( to_bits_ref(my_storage.my_value) ) - ); - } - - value_type load () const { - return load(); - } - - template - void store ( value_type value ) { - internal::atomic_load_store_traits::store( to_bits_ref(my_storage.my_value), to_bits(value)); - } - - void store ( value_type value ) { - store( value ); - } - -protected: - value_type store_with_release( value_type rhs ) { - //TODO: unify with store - __TBB_store_with_release( to_bits_ref(my_storage.my_value), to_bits(rhs) ); - return rhs; - } -}; - -//! Base class that provides basic functionality for atomic with fetch_and_add. -/** I is the underlying type. - D is the difference type. - StepType should be char if I is an integral type, and T if I is a T*. */ -template -struct atomic_impl_with_arithmetic: atomic_impl { -public: - typedef I value_type; -#if __TBB_ATOMIC_CTORS - atomic_impl_with_arithmetic() = default ; - constexpr atomic_impl_with_arithmetic(value_type value): atomic_impl(value){} -#endif - template - value_type fetch_and_add( D addend ) { - return value_type(internal::atomic_traits::fetch_and_add( &this->my_storage.my_value, addend*sizeof(StepType) )); - } - - value_type fetch_and_add( D addend ) { - return fetch_and_add(addend); - } - - template - value_type fetch_and_increment() { - return fetch_and_add(1); - } - - value_type fetch_and_increment() { - return fetch_and_add(1); - } - - template - value_type fetch_and_decrement() { - return fetch_and_add(__TBB_MINUS_ONE(D)); - } - - value_type fetch_and_decrement() { - return fetch_and_add(__TBB_MINUS_ONE(D)); - } - -public: - value_type operator+=( D value ) { - return fetch_and_add(value)+value; - } - - value_type operator-=( D value ) { - // Additive inverse of value computed using binary minus, - // instead of unary minus, for sake of avoiding compiler warnings. - return operator+=(D(0)-value); - } - - value_type operator++() { - return fetch_and_add(1)+1; - } - - value_type operator--() { - return fetch_and_add(__TBB_MINUS_ONE(D))-1; - } - - value_type operator++(int) { - return fetch_and_add(1); - } - - value_type operator--(int) { - return fetch_and_add(__TBB_MINUS_ONE(D)); - } -}; - -} /* Internal */ -//! @endcond - -//! Primary template for atomic. -/** See the Reference for details. - @ingroup synchronization */ -template -struct __TBB_DEPRECATED_VERBOSE_MSG("tbb::atomic is deprecated, use std::atomic") -atomic: internal::atomic_impl { -#if __TBB_ATOMIC_CTORS - atomic() = default; - constexpr atomic(T arg): internal::atomic_impl(arg) {} - constexpr atomic(const atomic& rhs): internal::atomic_impl(rhs) {} -#endif - T operator=( T rhs ) { - // "this" required here in strict ISO C++ because store_with_release is a dependent name - return this->store_with_release(rhs); - } - atomic& operator=( const atomic& rhs ) {this->store_with_release(rhs); return *this;} -}; - -#if __TBB_ATOMIC_CTORS - #define __TBB_DECL_ATOMIC(T) \ - template<> struct __TBB_DEPRECATED_VERBOSE_MSG("tbb::atomic is deprecated, use std::atomic") \ - atomic: internal::atomic_impl_with_arithmetic { \ - atomic() = default; \ - constexpr atomic(T arg): internal::atomic_impl_with_arithmetic(arg) {} \ - constexpr atomic(const atomic& rhs): \ - internal::atomic_impl_with_arithmetic(rhs) {} \ - \ - T operator=( T rhs ) {return store_with_release(rhs);} \ - atomic& operator=( const atomic& rhs ) {store_with_release(rhs); return *this;} \ - }; -#else - #define __TBB_DECL_ATOMIC(T) \ - template<> struct __TBB_DEPRECATED_VERBOSE_MSG("tbb::atomic is deprecated, use std::atomic") \ - atomic: internal::atomic_impl_with_arithmetic { \ - T operator=( T rhs ) {return store_with_release(rhs);} \ - atomic& operator=( const atomic& rhs ) {store_with_release(rhs); return *this;} \ - }; -#endif - -#if __TBB_64BIT_ATOMICS -//TODO: consider adding non-default (and atomic) copy constructor for 32bit platform -__TBB_DECL_ATOMIC(__TBB_LONG_LONG) -__TBB_DECL_ATOMIC(unsigned __TBB_LONG_LONG) -#else -// test_atomic will verify that sizeof(long long)==8 -#endif -__TBB_DECL_ATOMIC(long) -__TBB_DECL_ATOMIC(unsigned long) - -#if _MSC_VER && !_WIN64 -#if __TBB_ATOMIC_CTORS -/* Special version of __TBB_DECL_ATOMIC that avoids gratuitous warnings from cl /Wp64 option. - It is identical to __TBB_DECL_ATOMIC(unsigned) except that it replaces operator=(T) - with an operator=(U) that explicitly converts the U to a T. Types T and U should be - type synonyms on the platform. Type U should be the wider variant of T from the - perspective of /Wp64. */ -#define __TBB_DECL_ATOMIC_ALT(T,U) \ - template<> struct __TBB_DEPRECATED_VERBOSE_MSG("tbb::atomic is deprecated, use std::atomic") \ - atomic: internal::atomic_impl_with_arithmetic { \ - atomic() = default ; \ - constexpr atomic(T arg): internal::atomic_impl_with_arithmetic(arg) {} \ - constexpr atomic(const atomic& rhs): \ - internal::atomic_impl_with_arithmetic(rhs) {} \ - \ - T operator=( U rhs ) {return store_with_release(T(rhs));} \ - atomic& operator=( const atomic& rhs ) {store_with_release(rhs); return *this;} \ - }; -#else -#define __TBB_DECL_ATOMIC_ALT(T,U) \ - template<> struct __TBB_DEPRECATED_VERBOSE_MSG("tbb::atomic is deprecated, use std::atomic") \ - atomic: internal::atomic_impl_with_arithmetic { \ - T operator=( U rhs ) {return store_with_release(T(rhs));} \ - atomic& operator=( const atomic& rhs ) {store_with_release(rhs); return *this;} \ - }; -#endif -__TBB_DECL_ATOMIC_ALT(unsigned,size_t) -__TBB_DECL_ATOMIC_ALT(int,ptrdiff_t) -#else -__TBB_DECL_ATOMIC(unsigned) -__TBB_DECL_ATOMIC(int) -#endif /* _MSC_VER && !_WIN64 */ - -__TBB_DECL_ATOMIC(unsigned short) -__TBB_DECL_ATOMIC(short) -__TBB_DECL_ATOMIC(char) -__TBB_DECL_ATOMIC(signed char) -__TBB_DECL_ATOMIC(unsigned char) - -#if !_MSC_VER || defined(_NATIVE_WCHAR_T_DEFINED) -__TBB_DECL_ATOMIC(wchar_t) -#endif /* _MSC_VER||!defined(_NATIVE_WCHAR_T_DEFINED) */ - -//! Specialization for atomic with arithmetic and operator->. -template struct __TBB_DEPRECATED_VERBOSE_MSG("tbb::atomic is deprecated, use std::atomic") -atomic: internal::atomic_impl_with_arithmetic { -#if __TBB_ATOMIC_CTORS - atomic() = default ; - constexpr atomic(T* arg): internal::atomic_impl_with_arithmetic(arg) {} - constexpr atomic(const atomic& rhs): internal::atomic_impl_with_arithmetic(rhs) {} -#endif - T* operator=( T* rhs ) { - // "this" required here in strict ISO C++ because store_with_release is a dependent name - return this->store_with_release(rhs); - } - atomic& operator=( const atomic& rhs ) { - this->store_with_release(rhs); return *this; - } - T* operator->() const { - return (*this); - } -}; - -//! Specialization for atomic, for sake of not allowing arithmetic or operator->. -template<> struct __TBB_DEPRECATED_VERBOSE_MSG("tbb::atomic is deprecated, use std::atomic") -atomic: internal::atomic_impl { -#if __TBB_ATOMIC_CTORS - atomic() = default ; - constexpr atomic(void* arg): internal::atomic_impl(arg) {} - constexpr atomic(const atomic& rhs): internal::atomic_impl(rhs) {} -#endif - void* operator=( void* rhs ) { - // "this" required here in strict ISO C++ because store_with_release is a dependent name - return this->store_with_release(rhs); - } - atomic& operator=( const atomic& rhs ) { - this->store_with_release(rhs); return *this; - } -}; - -// Helpers to workaround ugly syntax of calling template member function of a -// template class with template argument dependent on template parameters. - -template -T load ( const atomic& a ) { return a.template load(); } - -template -void store ( atomic& a, T value ) { a.template store(value); } - -namespace interface6{ -//! Make an atomic for use in an initialization (list), as an alternative to zero-initialization or normal assignment. -template -atomic make_atomic(T t) { - atomic a; - store(a,t); - return a; -} -} -using interface6::make_atomic; - -namespace internal { -template -void swap(atomic & lhs, atomic & rhs){ - T tmp = load(lhs); - store(lhs,load(rhs)); - store(rhs,tmp); -} - -// only to aid in the gradual conversion of ordinary variables to proper atomics -template -inline atomic& as_atomic( T& t ) { - return (atomic&)t; -} -} // namespace tbb::internal - -} // namespace tbb - -#if _MSC_VER && !__INTEL_COMPILER - #pragma warning (pop) -#endif // warnings are restored - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_atomic_H_include_area - -#endif /* __TBB_atomic_H */ diff --git a/src/3rd party/serial/tbb/tbb_annotate.h b/src/3rd party/tbb/blocked_nd_range.h similarity index 53% rename from src/3rd party/serial/tbb/tbb_annotate.h rename to src/3rd party/tbb/blocked_nd_range.h index 6b79be2d01..70ca73af4b 100644 --- a/src/3rd party/serial/tbb/tbb_annotate.h +++ b/src/3rd party/tbb/blocked_nd_range.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2017-2024 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,19 +14,4 @@ limitations under the License. */ -#ifndef __TBB_annotate_H -#define __TBB_annotate_H - -// Macros used by the Intel(R) Parallel Advisor. -#ifdef __TBB_NORMAL_EXECUTION - #define ANNOTATE_SITE_BEGIN( site ) - #define ANNOTATE_SITE_END( site ) - #define ANNOTATE_TASK_BEGIN( task ) - #define ANNOTATE_TASK_END( task ) - #define ANNOTATE_LOCK_ACQUIRE( lock ) - #define ANNOTATE_LOCK_RELEASE( lock ) -#else - #include -#endif - -#endif /* __TBB_annotate_H */ +#include "../oneapi/tbb/blocked_nd_range.h" diff --git a/src/3rd party/tbb/blocked_range.h b/src/3rd party/tbb/blocked_range.h index d1ff1f4592..316ec01ba9 100644 --- a/src/3rd party/tbb/blocked_range.h +++ b/src/3rd party/tbb/blocked_range.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,155 +14,4 @@ limitations under the License. */ -#ifndef __TBB_blocked_range_H -#define __TBB_blocked_range_H - -#include "tbb_stddef.h" - -namespace tbb { - -namespace internal { - -// blocked_rangeNd_impl forward declaration in tbb::internal namespace to -// name it as a friend for a tbb::blocked_range. -template -class blocked_rangeNd_impl; - -} // namespace internal - -/** \page range_req Requirements on range concept - Class \c R implementing the concept of range must define: - - \code R::R( const R& ); \endcode Copy constructor - - \code R::~R(); \endcode Destructor - - \code bool R::is_divisible() const; \endcode True if range can be partitioned into two subranges - - \code bool R::empty() const; \endcode True if range is empty - - \code R::R( R& r, split ); \endcode Split range \c r into two subranges. -**/ - -//! A range over which to iterate. -/** @ingroup algorithms */ -template -class blocked_range { -public: - //! Type of a value - /** Called a const_iterator for sake of algorithms that need to treat a blocked_range - as an STL container. */ - typedef Value const_iterator; - - //! Type for size of a range - typedef std::size_t size_type; - -#if __TBB_DEPRECATED_BLOCKED_RANGE_DEFAULT_CTOR - //! Construct range with default-constructed values for begin, end, and grainsize. - /** Requires that Value have a default constructor. */ - blocked_range() : my_end(), my_begin(), my_grainsize() {} -#endif - - //! Construct range over half-open interval [begin,end), with the given grainsize. - blocked_range( Value begin_, Value end_, size_type grainsize_=1 ) : - my_end(end_), my_begin(begin_), my_grainsize(grainsize_) - { - __TBB_ASSERT( my_grainsize>0, "grainsize must be positive" ); - } - - //! Beginning of range. - const_iterator begin() const {return my_begin;} - - //! One past last value in range. - const_iterator end() const {return my_end;} - - //! Size of the range - /** Unspecified if end() - friend class blocked_range2d; - - template - friend class blocked_range3d; - - template - friend class internal::blocked_rangeNd_impl; -}; - -} // namespace tbb - -#endif /* __TBB_blocked_range_H */ +#include "../oneapi/tbb/blocked_range.h" diff --git a/src/3rd party/tbb/blocked_range2d.h b/src/3rd party/tbb/blocked_range2d.h index cd0fe1c3f8..1e13240787 100644 --- a/src/3rd party/tbb/blocked_range2d.h +++ b/src/3rd party/tbb/blocked_range2d.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,91 +14,4 @@ limitations under the License. */ -#ifndef __TBB_blocked_range2d_H -#define __TBB_blocked_range2d_H - -#include "tbb_stddef.h" -#include "blocked_range.h" - -namespace tbb { - -//! A 2-dimensional range that models the Range concept. -/** @ingroup algorithms */ -template -class blocked_range2d { -public: - //! Type for size of an iteration range - typedef blocked_range row_range_type; - typedef blocked_range col_range_type; - -private: - row_range_type my_rows; - col_range_type my_cols; - -public: - - blocked_range2d( RowValue row_begin, RowValue row_end, typename row_range_type::size_type row_grainsize, - ColValue col_begin, ColValue col_end, typename col_range_type::size_type col_grainsize ) : - my_rows(row_begin,row_end,row_grainsize), - my_cols(col_begin,col_end,col_grainsize) - {} - - blocked_range2d( RowValue row_begin, RowValue row_end, - ColValue col_begin, ColValue col_end ) : - my_rows(row_begin,row_end), - my_cols(col_begin,col_end) - {} - - //! True if range is empty - bool empty() const { - // Range is empty if at least one dimension is empty. - return my_rows.empty() || my_cols.empty(); - } - - //! True if range is divisible into two pieces. - bool is_divisible() const { - return my_rows.is_divisible() || my_cols.is_divisible(); - } - - blocked_range2d( blocked_range2d& r, split ) : - my_rows(r.my_rows), - my_cols(r.my_cols) - { - split split_obj; - do_split(r, split_obj); - } - -#if __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES - //! Static field to support proportional split - static const bool is_splittable_in_proportion = true; - - blocked_range2d( blocked_range2d& r, proportional_split& proportion ) : - my_rows(r.my_rows), - my_cols(r.my_cols) - { - do_split(r, proportion); - } -#endif /* __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES */ - - //! The rows of the iteration space - const row_range_type& rows() const {return my_rows;} - - //! The columns of the iteration space - const col_range_type& cols() const {return my_cols;} - -private: - - template - void do_split( blocked_range2d& r, Split& split_obj ) - { - if( my_rows.size()*double(my_cols.grainsize()) < my_cols.size()*double(my_rows.grainsize()) ) { - my_cols.my_begin = col_range_type::do_split(r.my_cols, split_obj); - } else { - my_rows.my_begin = row_range_type::do_split(r.my_rows, split_obj); - } - } -}; - -} // namespace tbb - -#endif /* __TBB_blocked_range2d_H */ +#include "../oneapi/tbb/blocked_range2d.h" diff --git a/src/3rd party/tbb/blocked_range3d.h b/src/3rd party/tbb/blocked_range3d.h index 5c6cf9f0e8..3321979660 100644 --- a/src/3rd party/tbb/blocked_range3d.h +++ b/src/3rd party/tbb/blocked_range3d.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,110 +14,4 @@ limitations under the License. */ -#ifndef __TBB_blocked_range3d_H -#define __TBB_blocked_range3d_H - -#include "tbb_stddef.h" -#include "blocked_range.h" - -namespace tbb { - -//! A 3-dimensional range that models the Range concept. -/** @ingroup algorithms */ -template -class blocked_range3d { -public: - //! Type for size of an iteration range - typedef blocked_range page_range_type; - typedef blocked_range row_range_type; - typedef blocked_range col_range_type; - -private: - page_range_type my_pages; - row_range_type my_rows; - col_range_type my_cols; - -public: - - blocked_range3d( PageValue page_begin, PageValue page_end, - RowValue row_begin, RowValue row_end, - ColValue col_begin, ColValue col_end ) : - my_pages(page_begin,page_end), - my_rows(row_begin,row_end), - my_cols(col_begin,col_end) - {} - - blocked_range3d( PageValue page_begin, PageValue page_end, typename page_range_type::size_type page_grainsize, - RowValue row_begin, RowValue row_end, typename row_range_type::size_type row_grainsize, - ColValue col_begin, ColValue col_end, typename col_range_type::size_type col_grainsize ) : - my_pages(page_begin,page_end,page_grainsize), - my_rows(row_begin,row_end,row_grainsize), - my_cols(col_begin,col_end,col_grainsize) - {} - - //! True if range is empty - bool empty() const { - // Range is empty if at least one dimension is empty. - return my_pages.empty() || my_rows.empty() || my_cols.empty(); - } - - //! True if range is divisible into two pieces. - bool is_divisible() const { - return my_pages.is_divisible() || my_rows.is_divisible() || my_cols.is_divisible(); - } - - blocked_range3d( blocked_range3d& r, split ) : - my_pages(r.my_pages), - my_rows(r.my_rows), - my_cols(r.my_cols) - { - split split_obj; - do_split(r, split_obj); - } - -#if __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES - //! Static field to support proportional split - static const bool is_splittable_in_proportion = true; - - blocked_range3d( blocked_range3d& r, proportional_split& proportion ) : - my_pages(r.my_pages), - my_rows(r.my_rows), - my_cols(r.my_cols) - { - do_split(r, proportion); - } -#endif /* __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES */ - - //! The pages of the iteration space - const page_range_type& pages() const {return my_pages;} - - //! The rows of the iteration space - const row_range_type& rows() const {return my_rows;} - - //! The columns of the iteration space - const col_range_type& cols() const {return my_cols;} - -private: - - template - void do_split( blocked_range3d& r, Split& split_obj) - { - if ( my_pages.size()*double(my_rows.grainsize()) < my_rows.size()*double(my_pages.grainsize()) ) { - if ( my_rows.size()*double(my_cols.grainsize()) < my_cols.size()*double(my_rows.grainsize()) ) { - my_cols.my_begin = col_range_type::do_split(r.my_cols, split_obj); - } else { - my_rows.my_begin = row_range_type::do_split(r.my_rows, split_obj); - } - } else { - if ( my_pages.size()*double(my_cols.grainsize()) < my_cols.size()*double(my_pages.grainsize()) ) { - my_cols.my_begin = col_range_type::do_split(r.my_cols, split_obj); - } else { - my_pages.my_begin = page_range_type::do_split(r.my_pages, split_obj); - } - } - } -}; - -} // namespace tbb - -#endif /* __TBB_blocked_range3d_H */ +#include "../oneapi/tbb/blocked_range3d.h" diff --git a/src/3rd party/tbb/blocked_rangeNd.h b/src/3rd party/tbb/blocked_rangeNd.h deleted file mode 100644 index b623d0021e..0000000000 --- a/src/3rd party/tbb/blocked_rangeNd.h +++ /dev/null @@ -1,150 +0,0 @@ -/* - Copyright (c) 2017-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_blocked_rangeNd_H -#define __TBB_blocked_rangeNd_H - -#if ! TBB_PREVIEW_BLOCKED_RANGE_ND - #error Set TBB_PREVIEW_BLOCKED_RANGE_ND to include blocked_rangeNd.h -#endif - -#include "tbb_config.h" - -// tbb::blocked_rangeNd requires C++11 support -#if __TBB_CPP11_PRESENT && __TBB_CPP11_ARRAY_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT - -#include "internal/_template_helpers.h" // index_sequence, make_index_sequence - -#include -#include // std::any_of -#include // std::is_same, std::enable_if - -#include "tbb/blocked_range.h" - -namespace tbb { -namespace internal { - -/* - The blocked_rangeNd_impl uses make_index_sequence to automatically generate a ctor with - exactly N arguments of the type tbb::blocked_range. Such ctor provides an opportunity - to use braced-init-list parameters to initialize each dimension. - Use of parameters, whose representation is a braced-init-list, but they're not - std::initializer_list or a reference to one, produces a non-deduced context - within template argument deduction. - - NOTE: blocked_rangeNd must be exactly a templated alias to the blocked_rangeNd_impl - (and not e.g. a derived class), otherwise it would need to declare its own ctor - facing the same problem that the impl class solves. -*/ - -template> -class blocked_rangeNd_impl; - -template -class blocked_rangeNd_impl> { -public: - //! Type of a value. - using value_type = Value; - -private: - - //! Helper type to construct range with N tbb::blocked_range objects. - template - using dim_type_helper = tbb::blocked_range; - -public: - blocked_rangeNd_impl() = delete; - - //! Constructs N-dimensional range over N half-open intervals each represented as tbb::blocked_range. - blocked_rangeNd_impl(const dim_type_helper&... args) : my_dims{ {args...} } {} - - //! Dimensionality of a range. - static constexpr unsigned int ndims() { return N; } - - //! Range in certain dimension. - const tbb::blocked_range& dim(unsigned int dimension) const { - __TBB_ASSERT(dimension < N, "out of bound"); - return my_dims[dimension]; - } - - //------------------------------------------------------------------------ - // Methods that implement Range concept - //------------------------------------------------------------------------ - - //! True if at least one dimension is empty. - bool empty() const { - return std::any_of(my_dims.begin(), my_dims.end(), [](const tbb::blocked_range& d) { - return d.empty(); - }); - } - - //! True if at least one dimension is divisible. - bool is_divisible() const { - return std::any_of(my_dims.begin(), my_dims.end(), [](const tbb::blocked_range& d) { - return d.is_divisible(); - }); - } - -#if __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES - //! Static field to support proportional split. - static const bool is_splittable_in_proportion = true; - - blocked_rangeNd_impl(blocked_rangeNd_impl& r, proportional_split proportion) : my_dims(r.my_dims) { - do_split(r, proportion); - } -#endif - - blocked_rangeNd_impl(blocked_rangeNd_impl& r, split proportion) : my_dims(r.my_dims) { - do_split(r, proportion); - } - -private: - __TBB_STATIC_ASSERT(N != 0, "zero dimensional blocked_rangeNd can't be constructed"); - - //! Ranges in each dimension. - std::array, N> my_dims; - - template - void do_split(blocked_rangeNd_impl& r, split_type proportion) { - __TBB_STATIC_ASSERT((is_same_type::value - || is_same_type::value), - "type of split object is incorrect"); - __TBB_ASSERT(r.is_divisible(), "can't split not divisible range"); - - auto my_it = std::max_element(my_dims.begin(), my_dims.end(), [](const tbb::blocked_range& first, const tbb::blocked_range& second) { - return (first.size() * second.grainsize() < second.size() * first.grainsize()); - }); - - auto r_it = r.my_dims.begin() + (my_it - my_dims.begin()); - - my_it->my_begin = tbb::blocked_range::do_split(*r_it, proportion); - - // (!(my_it->my_begin < r_it->my_end) && !(r_it->my_end < my_it->my_begin)) equals to - // (my_it->my_begin == r_it->my_end), but we can't use operator== due to Value concept - __TBB_ASSERT(!(my_it->my_begin < r_it->my_end) && !(r_it->my_end < my_it->my_begin), - "blocked_range has been split incorrectly"); - } -}; - -} // namespace internal - -template -using blocked_rangeNd = internal::blocked_rangeNd_impl; - -} // namespace tbb - -#endif /* __TBB_CPP11_PRESENT && __TBB_CPP11_ARRAY_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT */ -#endif /* __TBB_blocked_rangeNd_H */ diff --git a/src/3rd party/tbb/cache_aligned_allocator.h b/src/3rd party/tbb/cache_aligned_allocator.h index a9983298b0..2d3c66a74a 100644 --- a/src/3rd party/tbb/cache_aligned_allocator.h +++ b/src/3rd party/tbb/cache_aligned_allocator.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,196 +14,4 @@ limitations under the License. */ -#ifndef __TBB_cache_aligned_allocator_H -#define __TBB_cache_aligned_allocator_H - -#include -#include "tbb_stddef.h" -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC -#include // std::forward -#endif - -#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT -#include -#endif - -namespace tbb { - -//! @cond INTERNAL -namespace internal { - //! Cache/sector line size. - /** @ingroup memory_allocation */ - size_t __TBB_EXPORTED_FUNC NFS_GetLineSize(); - - //! Allocate memory on cache/sector line boundary. - /** @ingroup memory_allocation */ - void* __TBB_EXPORTED_FUNC NFS_Allocate( size_t n_element, size_t element_size, void* hint ); - - //! Free memory allocated by NFS_Allocate. - /** Freeing a NULL pointer is allowed, but has no effect. - @ingroup memory_allocation */ - void __TBB_EXPORTED_FUNC NFS_Free( void* ); -} -//! @endcond - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Workaround for erroneous "unreferenced parameter" warning in method destroy. - #pragma warning (push) - #pragma warning (disable: 4100) -#endif - -//! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 -/** The members are ordered the same way they are in section 20.4.1 - of the ISO C++ standard. - @ingroup memory_allocation */ -template -class cache_aligned_allocator { -public: - typedef typename internal::allocator_type::value_type value_type; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - template struct rebind { - typedef cache_aligned_allocator other; - }; - cache_aligned_allocator() throw() {} - cache_aligned_allocator( const cache_aligned_allocator& ) throw() {} - template cache_aligned_allocator(const cache_aligned_allocator&) throw() {} - - pointer address(reference x) const {return &x;} - const_pointer address(const_reference x) const {return &x;} - - //! Allocate space for n objects, starting on a cache/sector line. - pointer allocate( size_type n, const void* hint=0 ) { - // The "hint" argument is always ignored in NFS_Allocate thus const_cast shouldn't hurt - return pointer(internal::NFS_Allocate( n, sizeof(value_type), const_cast(hint) )); - } - - //! Free block of memory that starts on a cache line - void deallocate( pointer p, size_type ) { - internal::NFS_Free(p); - } - - //! Largest value for which method allocate might succeed. - size_type max_size() const throw() { - return (~size_t(0)-internal::NFS_MaxLineSize)/sizeof(value_type); - } - - //! Copy-construct value at location pointed to by p. -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - template - void construct(U *p, Args&&... args) - { ::new((void *)p) U(std::forward(args)...); } -#else // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC -#if __TBB_CPP11_RVALUE_REF_PRESENT - void construct( pointer p, value_type&& value ) {::new((void*)(p)) value_type(std::move(value));} -#endif - void construct( pointer p, const value_type& value ) {::new((void*)(p)) value_type(value);} -#endif // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - - //! Destroy value at location pointed to by p. - void destroy( pointer p ) {p->~value_type();} -}; - -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warning 4100 is back - -//! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1 -/** @ingroup memory_allocation */ -template<> -class cache_aligned_allocator { -public: - typedef void* pointer; - typedef const void* const_pointer; - typedef void value_type; - template struct rebind { - typedef cache_aligned_allocator other; - }; -}; - -template -inline bool operator==( const cache_aligned_allocator&, const cache_aligned_allocator& ) {return true;} - -template -inline bool operator!=( const cache_aligned_allocator&, const cache_aligned_allocator& ) {return false;} - -#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT - -//! C++17 memory resource wrapper to ensure cache line size alignment -class cache_aligned_resource : public std::pmr::memory_resource { -public: - cache_aligned_resource() : cache_aligned_resource(std::pmr::get_default_resource()) {} - explicit cache_aligned_resource(std::pmr::memory_resource* upstream) : m_upstream(upstream) {} - - std::pmr::memory_resource* upstream_resource() const { - return m_upstream; - } - -private: - //! We don't know what memory resource set. Use padding to guarantee alignment - void* do_allocate(size_t bytes, size_t alignment) override { - size_t cache_line_alignment = correct_alignment(alignment); - uintptr_t base = (uintptr_t)m_upstream->allocate(correct_size(bytes) + cache_line_alignment); - __TBB_ASSERT(base != 0, "Upstream resource returned NULL."); -#if _MSC_VER && !defined(__INTEL_COMPILER) - // unary minus operator applied to unsigned type, result still unsigned - #pragma warning(push) - #pragma warning(disable: 4146 4706) -#endif - // Round up to the next cache line (align the base address) - uintptr_t result = (base + cache_line_alignment) & -cache_line_alignment; -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning(pop) -#endif - // Record where block actually starts. - ((uintptr_t*)result)[-1] = base; - return (void*)result; - } - - void do_deallocate(void* ptr, size_t bytes, size_t alignment) override { - if (ptr) { - // Recover where block actually starts - uintptr_t base = ((uintptr_t*)ptr)[-1]; - m_upstream->deallocate((void*)base, correct_size(bytes) + correct_alignment(alignment)); - } - } - - bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override { - if (this == &other) { return true; } -#if __TBB_USE_OPTIONAL_RTTI - const cache_aligned_resource* other_res = dynamic_cast(&other); - return other_res && (this->upstream_resource() == other_res->upstream_resource()); -#else - return false; -#endif - } - - size_t correct_alignment(size_t alignment) { - __TBB_ASSERT(tbb::internal::is_power_of_two(alignment), "Alignment is not a power of 2"); -#if __TBB_CPP17_HW_INTERFERENCE_SIZE_PRESENT - size_t cache_line_size = std::hardware_destructive_interference_size; -#else - size_t cache_line_size = internal::NFS_GetLineSize(); -#endif - return alignment < cache_line_size ? cache_line_size : alignment; - } - - size_t correct_size(size_t bytes) { - // To handle the case, when small size requested. There could be not - // enough space to store the original pointer. - return bytes < sizeof(uintptr_t) ? sizeof(uintptr_t) : bytes; - } - - std::pmr::memory_resource* m_upstream; -}; - -#endif /* __TBB_CPP17_MEMORY_RESOURCE_PRESENT */ - -} // namespace tbb - -#endif /* __TBB_cache_aligned_allocator_H */ - +#include "../oneapi/tbb/cache_aligned_allocator.h" diff --git a/src/3rd party/tbb/collaborative_call_once.h b/src/3rd party/tbb/collaborative_call_once.h new file mode 100644 index 0000000000..68cbbcddbc --- /dev/null +++ b/src/3rd party/tbb/collaborative_call_once.h @@ -0,0 +1,17 @@ +/* + Copyright (c) 2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "../oneapi/tbb/collaborative_call_once.h" diff --git a/src/3rd party/tbb/combinable.h b/src/3rd party/tbb/combinable.h index 917c5fcc72..50295ec72a 100644 --- a/src/3rd party/tbb/combinable.h +++ b/src/3rd party/tbb/combinable.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,75 +14,4 @@ limitations under the License. */ -#ifndef __TBB_combinable_H -#define __TBB_combinable_H - -#define __TBB_combinable_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "enumerable_thread_specific.h" -#include "cache_aligned_allocator.h" - -namespace tbb { -/** \name combinable - **/ -//@{ -//! Thread-local storage with optional reduction -/** @ingroup containers */ - template - class combinable { - - private: - typedef typename tbb::cache_aligned_allocator my_alloc; - typedef typename tbb::enumerable_thread_specific my_ets_type; - my_ets_type my_ets; - - public: - - combinable() { } - - template - explicit combinable( finit _finit) : my_ets(_finit) { } - - //! destructor - ~combinable() { } - - combinable( const combinable& other) : my_ets(other.my_ets) { } - -#if __TBB_ETS_USE_CPP11 - combinable( combinable&& other) : my_ets( std::move(other.my_ets)) { } -#endif - - combinable & operator=( const combinable & other) { - my_ets = other.my_ets; - return *this; - } - -#if __TBB_ETS_USE_CPP11 - combinable & operator=( combinable && other) { - my_ets=std::move(other.my_ets); - return *this; - } -#endif - - void clear() { my_ets.clear(); } - - T& local() { return my_ets.local(); } - - T& local(bool & exists) { return my_ets.local(exists); } - - // combine_func_t has signature T(T,T) or T(const T&, const T&) - template - T combine(combine_func_t f_combine) { return my_ets.combine(f_combine); } - - // combine_func_t has signature void(T) or void(const T&) - template - void combine_each(combine_func_t f_combine) { my_ets.combine_each(f_combine); } - - }; -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_combinable_H_include_area - -#endif /* __TBB_combinable_H */ +#include "../oneapi/tbb/combinable.h" diff --git a/src/3rd party/tbb/compat/condition_variable b/src/3rd party/tbb/compat/condition_variable deleted file mode 100644 index e06324f04a..0000000000 --- a/src/3rd party/tbb/compat/condition_variable +++ /dev/null @@ -1,489 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include "../internal/_deprecated_header_message_guard.h" - -#if !defined(__TBB_show_deprecation_message_condition_variable_H) && defined(__TBB_show_deprecated_header_message) -#define __TBB_show_deprecation_message_condition_variable_H -#pragma message("TBB Warning: tbb/compat/condition_variable is deprecated. For details, please see Deprecated Features appendix in the TBB reference manual.") -#endif - -#if defined(__TBB_show_deprecated_header_message) -#undef __TBB_show_deprecated_header_message -#endif - -#ifndef __TBB_condition_variable_H -#define __TBB_condition_variable_H - -#define __TBB_condition_variable_H_include_area -#include "../internal/_warning_suppress_enable_notice.h" - -#if _WIN32||_WIN64 -#include "../machine/windows_api.h" - -namespace tbb { -namespace interface5 { -namespace internal { -struct condition_variable_using_event -{ - //! Event for blocking waiting threads. - HANDLE event; - //! Protects invariants involving n_waiters, release_count, and epoch. - CRITICAL_SECTION mutex; - //! Number of threads waiting on this condition variable - int n_waiters; - //! Number of threads remaining that should no longer wait on this condition variable. - int release_count; - //! To keep threads from waking up prematurely with earlier signals. - unsigned epoch; -}; -}}} // namespace tbb::interface5::internal - -#ifndef CONDITION_VARIABLE_INIT -typedef void* CONDITION_VARIABLE; -typedef CONDITION_VARIABLE* PCONDITION_VARIABLE; -#endif - -#else /* if not _WIN32||_WIN64 */ -#include // some systems need it for ETIMEDOUT -#include -#if __linux__ -#include -#else /* generic Unix */ -#include -#endif -#endif /* _WIN32||_WIN64 */ - -#include "../tbb_stddef.h" -#include "../mutex.h" -#include "../tbb_thread.h" -#include "../tbb_exception.h" -#include "../tbb_profiling.h" - -namespace tbb { - -namespace interface5 { - -// C++0x standard working draft 30.4.3 -// Lock tag types -struct __TBB_DEPRECATED_VERBOSE defer_lock_t { }; //! do not acquire ownership of the mutex -struct __TBB_DEPRECATED_VERBOSE try_to_lock_t { }; //! try to acquire ownership of the mutex without blocking -struct __TBB_DEPRECATED_VERBOSE adopt_lock_t { }; //! assume the calling thread has already -__TBB_DEPRECATED_VERBOSE const defer_lock_t defer_lock = {}; -__TBB_DEPRECATED_VERBOSE const try_to_lock_t try_to_lock = {}; -__TBB_DEPRECATED_VERBOSE const adopt_lock_t adopt_lock = {}; - -// C++0x standard working draft 30.4.3.1 -//! lock_guard -template -class __TBB_DEPRECATED_VERBOSE lock_guard : tbb::internal::no_copy { -public: - //! mutex type - typedef M mutex_type; - - //! Constructor - /** precondition: If mutex_type is not a recursive mutex, the calling thread - does not own the mutex m. */ - explicit lock_guard(mutex_type& m) : pm(m) {m.lock();} - - //! Adopt_lock constructor - /** precondition: the calling thread owns the mutex m. */ - lock_guard(mutex_type& m, adopt_lock_t) : pm(m) {} - - //! Destructor - ~lock_guard() { pm.unlock(); } -private: - mutex_type& pm; -}; - -// C++0x standard working draft 30.4.3.2 -//! unique_lock -template -class __TBB_DEPRECATED_VERBOSE unique_lock : tbb::internal::no_copy { - friend class condition_variable; -public: - typedef M mutex_type; - - // 30.4.3.2.1 construct/copy/destroy - // NB: Without constructors that take an r-value reference to a unique_lock, the following constructor is of little use. - //! Constructor - /** postcondition: pm==0 && owns==false */ - unique_lock() : pm(NULL), owns(false) {} - - //! Constructor - /** precondition: if mutex_type is not a recursive mutex, the calling thread - does not own the mutex m. If the precondition is not met, a deadlock occurs. - postcondition: pm==&m and owns==true */ - explicit unique_lock(mutex_type& m) : pm(&m) {m.lock(); owns=true;} - - //! Defer_lock constructor - /** postcondition: pm==&m and owns==false */ - unique_lock(mutex_type& m, defer_lock_t) : pm(&m), owns(false) {} - - //! Try_to_lock constructor - /** precondition: if mutex_type is not a recursive mutex, the calling thread - does not own the mutex m. If the precondition is not met, a deadlock occurs. - postcondition: pm==&m and owns==res where res is the value returned by - the call to m.try_lock(). */ - unique_lock(mutex_type& m, try_to_lock_t) : pm(&m) {owns = m.try_lock();} - - //! Adopt_lock constructor - /** precondition: the calling thread owns the mutex. If it does not, mutex->unlock() would fail. - postcondition: pm==&m and owns==true */ - unique_lock(mutex_type& m, adopt_lock_t) : pm(&m), owns(true) {} - - //! Timed unique_lock acquisition. - /** To avoid requiring support for namespace chrono, this method deviates from the working draft in that - it uses tbb::tick_count::interval_t to specify the time duration. */ - unique_lock(mutex_type& m, const tick_count::interval_t &i) : pm(&m) {owns = try_lock_for( i );} - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructor - /** postconditions: pm == src_p.pm and owns == src_p.owns (where src_p is the state of src just prior to this - construction), src.pm == 0 and src.owns == false. */ - unique_lock(unique_lock && src): pm(NULL), owns(false) {this->swap(src);} - - //! Move assignment - /** effects: If owns calls pm->unlock(). - Postconditions: pm == src_p.pm and owns == src_p.owns (where src_p is the state of src just prior to this - assignment), src.pm == 0 and src.owns == false. */ - unique_lock& operator=(unique_lock && src) { - if (owns) - this->unlock(); - pm = NULL; - this->swap(src); - return *this; - } -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - - //! Destructor - ~unique_lock() { if( owns ) pm->unlock(); } - - // 30.4.3.2.2 locking - //! Lock the mutex and own it. - void lock() { - if( pm ) { - if( !owns ) { - pm->lock(); - owns = true; - } else - throw_exception_v4( tbb::internal::eid_possible_deadlock ); - } else - throw_exception_v4( tbb::internal::eid_operation_not_permitted ); - __TBB_ASSERT( owns, NULL ); - } - - //! Try to lock the mutex. - /** If successful, note that this lock owns it. Otherwise, set it false. */ - bool try_lock() { - if( pm ) { - if( !owns ) - owns = pm->try_lock(); - else - throw_exception_v4( tbb::internal::eid_possible_deadlock ); - } else - throw_exception_v4( tbb::internal::eid_operation_not_permitted ); - return owns; - } - - //! Try to lock the mutex. - bool try_lock_for( const tick_count::interval_t &i ); - - //! Unlock the mutex - /** And note that this lock no longer owns it. */ - void unlock() { - if( owns ) { - pm->unlock(); - owns = false; - } else - throw_exception_v4( tbb::internal::eid_operation_not_permitted ); - __TBB_ASSERT( !owns, NULL ); - } - - // 30.4.3.2.3 modifiers - //! Swap the two unique locks - void swap(unique_lock& u) { - mutex_type* t_pm = u.pm; u.pm = pm; pm = t_pm; - bool t_owns = u.owns; u.owns = owns; owns = t_owns; - } - - //! Release control over the mutex. - mutex_type* release() { - mutex_type* o_pm = pm; - pm = NULL; - owns = false; - return o_pm; - } - - // 30.4.3.2.4 observers - //! Does this lock own the mutex? - bool owns_lock() const { return owns; } - - // TODO: Un-comment 'explicit' when the last non-C++0x compiler support is dropped - //! Does this lock own the mutex? - /*explicit*/ operator bool() const { return owns; } - - //! Return the mutex that this lock currently has. - mutex_type* mutex() const { return pm; } - -private: - mutex_type* pm; - bool owns; -}; - -template -__TBB_DEPRECATED_VERBOSE bool unique_lock::try_lock_for( const tick_count::interval_t &i) -{ - const int unique_lock_tick = 100; /* microseconds; 0.1 milliseconds */ - // the smallest wait-time is 0.1 milliseconds. - bool res = pm->try_lock(); - int duration_in_micro; - if( !res && (duration_in_micro=int(i.seconds()*1e6))>unique_lock_tick ) { - tick_count::interval_t i_100( double(unique_lock_tick)/1e6 /* seconds */); // 100 microseconds = 0.1*10E-3 - do { - this_tbb_thread::sleep(i_100); // sleep for 100 micro seconds - duration_in_micro -= unique_lock_tick; - res = pm->try_lock(); - } while( !res && duration_in_micro>unique_lock_tick ); - } - return (owns=res); -} - -//! Swap the two unique locks that have the mutexes of same type -template -void swap(unique_lock& x, unique_lock& y) { x.swap( y ); } - -namespace internal { - -#if _WIN32||_WIN64 -union condvar_impl_t { - condition_variable_using_event cv_event; - CONDITION_VARIABLE cv_native; -}; -void __TBB_EXPORTED_FUNC internal_initialize_condition_variable( condvar_impl_t& cv ); -void __TBB_EXPORTED_FUNC internal_destroy_condition_variable( condvar_impl_t& cv ); -void __TBB_EXPORTED_FUNC internal_condition_variable_notify_one( condvar_impl_t& cv ); -void __TBB_EXPORTED_FUNC internal_condition_variable_notify_all( condvar_impl_t& cv ); -bool __TBB_EXPORTED_FUNC internal_condition_variable_wait( condvar_impl_t& cv, mutex* mtx, const tick_count::interval_t* i = NULL ); - -#else /* if !(_WIN32||_WIN64), i.e., POSIX threads */ -typedef pthread_cond_t condvar_impl_t; -#endif - -} // namespace internal - -//! cv_status -/** C++0x standard working draft 30.5 */ -enum cv_status { no_timeout, timeout }; - -//! condition variable -/** C++0x standard working draft 30.5.1 - @ingroup synchronization */ -class __TBB_DEPRECATED_VERBOSE condition_variable : tbb::internal::no_copy { -public: - //! Constructor - condition_variable() { -#if _WIN32||_WIN64 - internal_initialize_condition_variable( my_cv ); -#else - pthread_cond_init( &my_cv, NULL ); -#endif - } - - //! Destructor - ~condition_variable() { - //precondition: There shall be no thread blocked on *this. -#if _WIN32||_WIN64 - internal_destroy_condition_variable( my_cv ); -#else - pthread_cond_destroy( &my_cv ); -#endif - } - - //! Notify one thread and wake it up - void notify_one() { -#if _WIN32||_WIN64 - internal_condition_variable_notify_one( my_cv ); -#else - pthread_cond_signal( &my_cv ); -#endif - } - - //! Notify all threads - void notify_all() { -#if _WIN32||_WIN64 - internal_condition_variable_notify_all( my_cv ); -#else - pthread_cond_broadcast( &my_cv ); -#endif - } - - //! Release the mutex associated with the lock and wait on this condition variable - void wait(unique_lock& lock); - - //! Wait on this condition variable while pred is false - template - void wait(unique_lock& lock, Predicate pred) { - while( !pred() ) - wait( lock ); - } - - //! Timed version of wait() - cv_status wait_for(unique_lock& lock, const tick_count::interval_t &i ); - - //! Timed version of the predicated wait - /** The loop terminates when pred() returns true or when the time duration specified by rel_time (i) has elapsed. */ - template - bool wait_for(unique_lock& lock, const tick_count::interval_t &i, Predicate pred) - { - while( !pred() ) { - cv_status st = wait_for( lock, i ); - if( st==timeout ) - return pred(); - } - return true; - } - - // C++0x standard working draft. 30.2.3 - typedef internal::condvar_impl_t* native_handle_type; - - native_handle_type native_handle() { return (native_handle_type) &my_cv; } - -private: - internal::condvar_impl_t my_cv; -}; - - -#if _WIN32||_WIN64 -inline void condition_variable::wait( unique_lock& lock ) -{ - __TBB_ASSERT( lock.owns, NULL ); - lock.owns = false; - if( !internal_condition_variable_wait( my_cv, lock.mutex() ) ) { - int ec = GetLastError(); - // on Windows 7, SleepConditionVariableCS() may return ERROR_TIMEOUT while the doc says it returns WAIT_TIMEOUT - __TBB_ASSERT_EX( ec!=WAIT_TIMEOUT&&ec!=ERROR_TIMEOUT, NULL ); - lock.owns = true; - throw_exception_v4( tbb::internal::eid_condvar_wait_failed ); - } - lock.owns = true; -} - -inline cv_status condition_variable::wait_for( unique_lock& lock, const tick_count::interval_t& i ) -{ - cv_status rc = no_timeout; - __TBB_ASSERT( lock.owns, NULL ); - lock.owns = false; - // condvar_wait could be SleepConditionVariableCS (or SleepConditionVariableSRW) or our own pre-vista cond_var_wait() - if( !internal_condition_variable_wait( my_cv, lock.mutex(), &i ) ) { - int ec = GetLastError(); - if( ec==WAIT_TIMEOUT || ec==ERROR_TIMEOUT ) - rc = timeout; - else { - lock.owns = true; - throw_exception_v4( tbb::internal::eid_condvar_wait_failed ); - } - } - lock.owns = true; - return rc; -} - -#else /* !(_WIN32||_WIN64) */ -inline void condition_variable::wait( unique_lock& lock ) -{ - __TBB_ASSERT( lock.owns, NULL ); - lock.owns = false; - if( pthread_cond_wait( &my_cv, lock.mutex()->native_handle() ) ) { - lock.owns = true; - throw_exception_v4( tbb::internal::eid_condvar_wait_failed ); - } - // upon successful return, the mutex has been locked and is owned by the calling thread. - lock.owns = true; -} - -inline cv_status condition_variable::wait_for( unique_lock& lock, const tick_count::interval_t& i ) -{ -#if __linux__ - struct timespec req; - double sec = i.seconds(); - clock_gettime( CLOCK_REALTIME, &req ); - req.tv_sec += static_cast(sec); - req.tv_nsec += static_cast( (sec - static_cast(sec))*1e9 ); -#else /* generic Unix */ - struct timeval tv; - struct timespec req; - double sec = i.seconds(); - int status = gettimeofday(&tv, NULL); - __TBB_ASSERT_EX( status==0, "gettimeofday failed" ); - req.tv_sec = tv.tv_sec + static_cast(sec); - req.tv_nsec = tv.tv_usec*1000 + static_cast( (sec - static_cast(sec))*1e9 ); -#endif /*(choice of OS) */ - if( req.tv_nsec>=1e9 ) { - req.tv_sec += 1; - req.tv_nsec -= static_cast(1e9); - } - __TBB_ASSERT( 0<=req.tv_nsec && req.tv_nsec<1e9, NULL ); - - int ec; - cv_status rc = no_timeout; - __TBB_ASSERT( lock.owns, NULL ); - lock.owns = false; - if( ( ec=pthread_cond_timedwait( &my_cv, lock.mutex()->native_handle(), &req ) ) ) { - if( ec==ETIMEDOUT ) - rc = timeout; - else { - __TBB_ASSERT( lock.try_lock()==false, NULL ); - lock.owns = true; - throw_exception_v4( tbb::internal::eid_condvar_wait_failed ); - } - } - lock.owns = true; - return rc; -} -#endif /* !(_WIN32||_WIN64) */ - -} // namespace interface5 - -__TBB_DEFINE_PROFILING_SET_NAME(interface5::condition_variable) - -} // namespace tbb - -#if TBB_IMPLEMENT_CPP0X - -namespace std { - -using tbb::interface5::defer_lock_t; -using tbb::interface5::try_to_lock_t; -using tbb::interface5::adopt_lock_t; -using tbb::interface5::defer_lock; -using tbb::interface5::try_to_lock; -using tbb::interface5::adopt_lock; -using tbb::interface5::lock_guard; -using tbb::interface5::unique_lock; -using tbb::interface5::swap; /* this is for void std::swap(unique_lock&,unique_lock&) */ -using tbb::interface5::condition_variable; -using tbb::interface5::cv_status; -using tbb::interface5::timeout; -using tbb::interface5::no_timeout; - -} // namespace std - -#endif /* TBB_IMPLEMENT_CPP0X */ - -#include "../internal/_warning_suppress_disable_notice.h" -#undef __TBB_condition_variable_H_include_area - -#endif /* __TBB_condition_variable_H */ diff --git a/src/3rd party/tbb/compat/ppl.h b/src/3rd party/tbb/compat/ppl.h deleted file mode 100644 index a3ca67ba16..0000000000 --- a/src/3rd party/tbb/compat/ppl.h +++ /dev/null @@ -1,75 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include "../internal/_deprecated_header_message_guard.h" - -#if !defined(__TBB_show_deprecation_message_ppl_H) && defined(__TBB_show_deprecated_header_message) -#define __TBB_show_deprecation_message_ppl_H -#pragma message("TBB Warning: tbb/compat/ppl.h is deprecated. For details, please see Deprecated Features appendix in the TBB reference manual.") -#endif - -#if defined(__TBB_show_deprecated_header_message) -#undef __TBB_show_deprecated_header_message -#endif - -#ifndef __TBB_compat_ppl_H -#define __TBB_compat_ppl_H - -#define __TBB_ppl_H_include_area -#include "../internal/_warning_suppress_enable_notice.h" - -#include "../task_group.h" -#include "../parallel_invoke.h" -#include "../parallel_for_each.h" -#include "../parallel_for.h" -#include "../tbb_exception.h" -#include "../critical_section.h" -#include "../reader_writer_lock.h" -#include "../combinable.h" - -namespace Concurrency { - -#if __TBB_TASK_GROUP_CONTEXT - using tbb::task_handle; - using tbb::task_group_status; - using tbb::task_group; - using tbb::structured_task_group; - using tbb::invalid_multiple_scheduling; - using tbb::missing_wait; - using tbb::make_task; - - using tbb::not_complete; - using tbb::complete; - using tbb::canceled; - - using tbb::is_current_task_group_canceling; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - - using tbb::parallel_invoke; - using tbb::strict_ppl::parallel_for; - using tbb::parallel_for_each; - using tbb::critical_section; - using tbb::reader_writer_lock; - using tbb::combinable; - - using tbb::improper_lock; - -} // namespace Concurrency - -#include "../internal/_warning_suppress_disable_notice.h" -#undef __TBB_ppl_H_include_area - -#endif /* __TBB_compat_ppl_H */ diff --git a/src/3rd party/tbb/compat/thread b/src/3rd party/tbb/compat/thread deleted file mode 100644 index c41bd03657..0000000000 --- a/src/3rd party/tbb/compat/thread +++ /dev/null @@ -1,73 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include "../internal/_deprecated_header_message_guard.h" - -#if !defined(__TBB_show_deprecation_message_thread_H) && defined(__TBB_show_deprecated_header_message) -#define __TBB_show_deprecation_message_thread_H -#pragma message("TBB Warning: tbb/compat/thread is deprecated. For details, please see Deprecated Features appendix in the TBB reference manual.") -#endif - -#if defined(__TBB_show_deprecated_header_message) -#undef __TBB_show_deprecated_header_message -#endif - -#ifndef __TBB_thread_H -#define __TBB_thread_H - -#define __TBB_thread_H_include_area -#include "../internal/_warning_suppress_enable_notice.h" - -#include "../tbb_config.h" - -#if TBB_IMPLEMENT_CPP0X - -#include "../tbb_thread.h" - -namespace std { - -typedef tbb::tbb_thread thread; - -namespace this_thread { - using tbb::this_tbb_thread::get_id; - using tbb::this_tbb_thread::yield; - - __TBB_DEPRECATED_VERBOSE inline void sleep_for(const tbb::tick_count::interval_t& rel_time) { - tbb::internal::thread_sleep_v3( rel_time ); - } -} - -} // namespace std - -#else /* TBB_IMPLEMENT_CPP0X */ - -#define __TBB_COMPAT_THREAD_RECURSION_PROTECTOR 1 -#include -#undef __TBB_COMPAT_THREAD_RECURSION_PROTECTOR - -#endif /* TBB_IMPLEMENT_CPP0X */ - -#include "../internal/_warning_suppress_disable_notice.h" -#undef __TBB_thread_H_include_area - -#else /* __TBB_thread_H */ - -#if __TBB_COMPAT_THREAD_RECURSION_PROTECTOR -#error The tbb/compat/thread header attempts to include itself. \ - Please make sure that {TBBROOT}/include/tbb/compat is NOT in include paths. -#endif - -#endif /* __TBB_thread_H */ diff --git a/src/3rd party/tbb/compat/tuple b/src/3rd party/tbb/compat/tuple deleted file mode 100644 index bd9da1713c..0000000000 --- a/src/3rd party/tbb/compat/tuple +++ /dev/null @@ -1,501 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include "../internal/_deprecated_header_message_guard.h" - -#if !defined(__TBB_show_deprecation_message_tuple_H) && defined(__TBB_show_deprecated_header_message) -#define __TBB_show_deprecation_message_tuple_H -#pragma message("TBB Warning: tbb/compat/tuple is deprecated. For details, please see Deprecated Features appendix in the TBB reference manual.") -#endif - -#if defined(__TBB_show_deprecated_header_message) -#undef __TBB_show_deprecated_header_message -#endif - -#ifndef __TBB_tuple_H -#define __TBB_tuple_H - -#define __TBB_tuple_H_include_area -#include "../internal/_warning_suppress_enable_notice.h" - -#include -#include "../tbb_stddef.h" - -// build preprocessor variables for varying number of arguments -// Need the leading comma so the empty __TBB_T_PACK will not cause a syntax error. -#if __TBB_VARIADIC_MAX <= 5 -#define __TBB_T_PACK -#define __TBB_U_PACK -#define __TBB_TYPENAME_T_PACK -#define __TBB_TYPENAME_U_PACK -#define __TBB_NULL_TYPE_PACK -#define __TBB_REF_T_PARAM_PACK -#define __TBB_CONST_REF_T_PARAM_PACK -#define __TBB_T_PARAM_LIST_PACK -#define __TBB_CONST_NULL_REF_PACK -// -#elif __TBB_VARIADIC_MAX == 6 -#define __TBB_T_PACK ,__T5 -#define __TBB_U_PACK ,__U5 -#define __TBB_TYPENAME_T_PACK , typename __T5 -#define __TBB_TYPENAME_U_PACK , typename __U5 -#define __TBB_NULL_TYPE_PACK , null_type -#define __TBB_REF_T_PARAM_PACK ,__T5& t5 -#define __TBB_CONST_REF_T_PARAM_PACK ,const __T5& t5 -#define __TBB_T_PARAM_LIST_PACK ,t5 -#define __TBB_CONST_NULL_REF_PACK , const null_type& -// -#elif __TBB_VARIADIC_MAX == 7 -#define __TBB_T_PACK ,__T5, __T6 -#define __TBB_U_PACK ,__U5, __U6 -#define __TBB_TYPENAME_T_PACK , typename __T5 , typename __T6 -#define __TBB_TYPENAME_U_PACK , typename __U5 , typename __U6 -#define __TBB_NULL_TYPE_PACK , null_type, null_type -#define __TBB_REF_T_PARAM_PACK ,__T5& t5, __T6& t6 -#define __TBB_CONST_REF_T_PARAM_PACK ,const __T5& t5, const __T6& t6 -#define __TBB_T_PARAM_LIST_PACK ,t5 ,t6 -#define __TBB_CONST_NULL_REF_PACK , const null_type&, const null_type& -// -#elif __TBB_VARIADIC_MAX == 8 -#define __TBB_T_PACK ,__T5, __T6, __T7 -#define __TBB_U_PACK ,__U5, __U6, __U7 -#define __TBB_TYPENAME_T_PACK , typename __T5 , typename __T6, typename __T7 -#define __TBB_TYPENAME_U_PACK , typename __U5 , typename __U6, typename __U7 -#define __TBB_NULL_TYPE_PACK , null_type, null_type, null_type -#define __TBB_REF_T_PARAM_PACK ,__T5& t5, __T6& t6, __T7& t7 -#define __TBB_CONST_REF_T_PARAM_PACK , const __T5& t5, const __T6& t6, const __T7& t7 -#define __TBB_T_PARAM_LIST_PACK ,t5 ,t6 ,t7 -#define __TBB_CONST_NULL_REF_PACK , const null_type&, const null_type&, const null_type& -// -#elif __TBB_VARIADIC_MAX == 9 -#define __TBB_T_PACK ,__T5, __T6, __T7, __T8 -#define __TBB_U_PACK ,__U5, __U6, __U7, __U8 -#define __TBB_TYPENAME_T_PACK , typename __T5, typename __T6, typename __T7, typename __T8 -#define __TBB_TYPENAME_U_PACK , typename __U5, typename __U6, typename __U7, typename __U8 -#define __TBB_NULL_TYPE_PACK , null_type, null_type, null_type, null_type -#define __TBB_REF_T_PARAM_PACK ,__T5& t5, __T6& t6, __T7& t7, __T8& t8 -#define __TBB_CONST_REF_T_PARAM_PACK , const __T5& t5, const __T6& t6, const __T7& t7, const __T8& t8 -#define __TBB_T_PARAM_LIST_PACK ,t5 ,t6 ,t7 ,t8 -#define __TBB_CONST_NULL_REF_PACK , const null_type&, const null_type&, const null_type&, const null_type& -// -#elif __TBB_VARIADIC_MAX >= 10 -#define __TBB_T_PACK ,__T5, __T6, __T7, __T8, __T9 -#define __TBB_U_PACK ,__U5, __U6, __U7, __U8, __U9 -#define __TBB_TYPENAME_T_PACK , typename __T5, typename __T6, typename __T7, typename __T8, typename __T9 -#define __TBB_TYPENAME_U_PACK , typename __U5, typename __U6, typename __U7, typename __U8, typename __U9 -#define __TBB_NULL_TYPE_PACK , null_type, null_type, null_type, null_type, null_type -#define __TBB_REF_T_PARAM_PACK ,__T5& t5, __T6& t6, __T7& t7, __T8& t8, __T9& t9 -#define __TBB_CONST_REF_T_PARAM_PACK , const __T5& t5, const __T6& t6, const __T7& t7, const __T8& t8, const __T9& t9 -#define __TBB_T_PARAM_LIST_PACK ,t5 ,t6 ,t7 ,t8 ,t9 -#define __TBB_CONST_NULL_REF_PACK , const null_type&, const null_type&, const null_type&, const null_type&, const null_type& -#endif - - - -namespace tbb { -namespace interface5 { - -namespace internal { -struct null_type { }; -} -using internal::null_type; - -// tuple forward declaration -template = 6 -, typename __T5=null_type -#if __TBB_VARIADIC_MAX >= 7 -, typename __T6=null_type -#if __TBB_VARIADIC_MAX >= 8 -, typename __T7=null_type -#if __TBB_VARIADIC_MAX >= 9 -, typename __T8=null_type -#if __TBB_VARIADIC_MAX >= 10 -, typename __T9=null_type -#endif -#endif -#endif -#endif -#endif -> -class tuple; - -namespace internal { - -// const null_type temp -inline const null_type cnull() { return null_type(); } - -// cons forward declaration -template struct cons; - -// type of a component of the cons -template -struct component { - typedef typename __T::tail_type next; - typedef typename component<__N-1,next>::type type; -}; - -template -struct component<0,__T> { - typedef typename __T::head_type type; -}; - -template<> -struct component<0,null_type> { - typedef null_type type; -}; - -// const version of component - -template -struct component<__N, const __T> -{ - typedef typename __T::tail_type next; - typedef const typename component<__N-1,next>::type type; -}; - -template -struct component<0, const __T> -{ - typedef const typename __T::head_type type; -}; - - -// helper class for getting components of cons -template< int __N> -struct get_helper { -template -inline static typename component<__N, cons<__HT,__TT> >::type& get(cons<__HT,__TT>& ti) { - return get_helper<__N-1>::get(ti.tail); -} -template -inline static typename component<__N, cons<__HT,__TT> >::type const& get(const cons<__HT,__TT>& ti) { - return get_helper<__N-1>::get(ti.tail); -} -}; - -template<> -struct get_helper<0> { -template -inline static typename component<0, cons<__HT,__TT> >::type& get(cons<__HT,__TT>& ti) { - return ti.head; -} -template -inline static typename component<0, cons<__HT,__TT> >::type const& get(const cons<__HT,__TT>& ti) { - return ti.head; -} -}; - -// traits adaptor -template -struct tuple_traits { - typedef cons <__T0, typename tuple_traits<__T1, __T2, __T3, __T4 __TBB_T_PACK , null_type>::U > U; -}; - -template -struct tuple_traits<__T0, null_type, null_type, null_type, null_type __TBB_NULL_TYPE_PACK > { - typedef cons<__T0, null_type> U; -}; - -template<> -struct tuple_traits { - typedef null_type U; -}; - - -// core cons defs -template -struct cons{ - - typedef __HT head_type; - typedef __TT tail_type; - - head_type head; - tail_type tail; - - static const int length = 1 + tail_type::length; - - // default constructors - explicit cons() : head(), tail() { } - - // non-default constructors - cons(head_type& h, const tail_type& t) : head(h), tail(t) { } - - template - cons(const __T0& t0, const __T1& t1, const __T2& t2, const __T3& t3, const __T4& t4 __TBB_CONST_REF_T_PARAM_PACK) : - head(t0), tail(t1, t2, t3, t4 __TBB_T_PARAM_LIST_PACK, cnull()) { } - - template - cons(__T0& t0, __T1& t1, __T2& t2, __T3& t3, __T4& t4 __TBB_REF_T_PARAM_PACK) : - head(t0), tail(t1, t2, t3, t4 __TBB_T_PARAM_LIST_PACK , cnull()) { } - - template - cons(const cons<__HT1,__TT1>& other) : head(other.head), tail(other.tail) { } - - cons& operator=(const cons& other) { head = other.head; tail = other.tail; return *this; } - - friend bool operator==(const cons& me, const cons& other) { - return me.head == other.head && me.tail == other.tail; - } - friend bool operator<(const cons& me, const cons& other) { - return me.head < other.head || (!(other.head < me.head) && me.tail < other.tail); - } - friend bool operator>(const cons& me, const cons& other) { return other=(const cons& me, const cons& other) { return !(meother); } - - template - friend bool operator==(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { - return me.head == other.head && me.tail == other.tail; - } - - template - friend bool operator<(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { - return me.head < other.head || (!(other.head < me.head) && me.tail < other.tail); - } - - template - friend bool operator>(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { return other - friend bool operator!=(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { return !(me==other); } - - template - friend bool operator>=(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { return !(me - friend bool operator<=(const cons<__HT,__TT>& me, const cons<__HT1,__TT1>& other) { return !(me>other); } - - -}; // cons - - -template -struct cons<__HT,null_type> { - - typedef __HT head_type; - typedef null_type tail_type; - - head_type head; - - static const int length = 1; - - // default constructor - cons() : head() { /*std::cout << "default constructor 1\n";*/ } - - cons(const null_type&, const null_type&, const null_type&, const null_type&, const null_type& __TBB_CONST_NULL_REF_PACK) : head() { /*std::cout << "default constructor 2\n";*/ } - - // non-default constructor - template - cons(__T1& t1, const null_type&, const null_type&, const null_type&, const null_type& __TBB_CONST_NULL_REF_PACK) : head(t1) { /*std::cout << "non-default a1, t1== " << t1 << "\n";*/} - - cons(head_type& h, const null_type& = null_type() ) : head(h) { } - cons(const head_type& t0, const null_type&, const null_type&, const null_type&, const null_type& __TBB_CONST_NULL_REF_PACK) : head(t0) { } - - // converting constructor - template - cons(__HT1 h1, const null_type&, const null_type&, const null_type&, const null_type& __TBB_CONST_NULL_REF_PACK) : head(h1) { } - - // copy constructor - template - cons( const cons<__HT1, null_type>& other) : head(other.head) { } - - // assignment operator - cons& operator=(const cons& other) { head = other.head; return *this; } - - friend bool operator==(const cons& me, const cons& other) { return me.head == other.head; } - friend bool operator<(const cons& me, const cons& other) { return me.head < other.head; } - friend bool operator>(const cons& me, const cons& other) { return otherother); } - friend bool operator>=(const cons& me, const cons& other) {return !(me - friend bool operator==(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { - return me.head == other.head; - } - - template - friend bool operator<(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { - return me.head < other.head; - } - - template - friend bool operator>(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { return other - friend bool operator!=(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { return !(me==other); } - - template - friend bool operator<=(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { return !(me>other); } - - template - friend bool operator>=(const cons<__HT,null_type>& me, const cons<__HT1,null_type>& other) { return !(me -struct cons { typedef null_type tail_type; static const int length = 0; }; - -// wrapper for default constructor -template -inline const __T wrap_dcons(__T*) { return __T(); } - -} // namespace internal - -// tuple definition -template -class __TBB_DEPRECATED_VERBOSE tuple : public internal::tuple_traits<__T0, __T1, __T2, __T3, __T4 __TBB_T_PACK >::U { - // friends - template friend class tuple_size; - template friend struct tuple_element; - - // stl components - typedef tuple<__T0,__T1,__T2,__T3,__T4 __TBB_T_PACK > value_type; - typedef value_type *pointer; - typedef const value_type *const_pointer; - typedef value_type &reference; - typedef const value_type &const_reference; - typedef size_t size_type; - - typedef typename internal::tuple_traits<__T0,__T1,__T2,__T3, __T4 __TBB_T_PACK >::U my_cons; - -public: - __TBB_DEPRECATED_VERBOSE tuple(const __T0& t0=internal::wrap_dcons((__T0*)NULL) - ,const __T1& t1=internal::wrap_dcons((__T1*)NULL) - ,const __T2& t2=internal::wrap_dcons((__T2*)NULL) - ,const __T3& t3=internal::wrap_dcons((__T3*)NULL) - ,const __T4& t4=internal::wrap_dcons((__T4*)NULL) -#if __TBB_VARIADIC_MAX >= 6 - ,const __T5& t5=internal::wrap_dcons((__T5*)NULL) -#if __TBB_VARIADIC_MAX >= 7 - ,const __T6& t6=internal::wrap_dcons((__T6*)NULL) -#if __TBB_VARIADIC_MAX >= 8 - ,const __T7& t7=internal::wrap_dcons((__T7*)NULL) -#if __TBB_VARIADIC_MAX >= 9 - ,const __T8& t8=internal::wrap_dcons((__T8*)NULL) -#if __TBB_VARIADIC_MAX >= 10 - ,const __T9& t9=internal::wrap_dcons((__T9*)NULL) -#endif -#endif -#endif -#endif -#endif - ) : - my_cons(t0,t1,t2,t3,t4 __TBB_T_PARAM_LIST_PACK) { } - - template - struct internal_tuple_element { - typedef typename internal::component<__N,my_cons>::type type; - }; - - template - typename internal_tuple_element<__N>::type& get() { return internal::get_helper<__N>::get(*this); } - - template - typename internal_tuple_element<__N>::type const& get() const { return internal::get_helper<__N>::get(*this); } - - template - tuple& operator=(const internal::cons<__U1,__U2>& other) { - my_cons::operator=(other); - return *this; - } - - template - tuple& operator=(const std::pair<__U1,__U2>& other) { - // __TBB_ASSERT(tuple_size::value == 2, "Invalid size for pair to tuple assignment"); - this->head = other.first; - this->tail.head = other.second; - return *this; - } - - friend bool operator==(const tuple& me, const tuple& other) {return static_cast(me)==(other);} - friend bool operator<(const tuple& me, const tuple& other) {return static_cast(me)<(other);} - friend bool operator>(const tuple& me, const tuple& other) {return static_cast(me)>(other);} - friend bool operator!=(const tuple& me, const tuple& other) {return static_cast(me)!=(other);} - friend bool operator>=(const tuple& me, const tuple& other) {return static_cast(me)>=(other);} - friend bool operator<=(const tuple& me, const tuple& other) {return static_cast(me)<=(other);} - -}; // tuple - -// empty tuple -template<> -class __TBB_DEPRECATED_VERBOSE tuple : public null_type { -}; - -// helper classes - -template < typename __T> -class tuple_size { -public: - static const size_t value = 1 + tuple_size::value; -}; - -template <> -class tuple_size > { -public: - static const size_t value = 0; -}; - -template <> -class tuple_size { -public: - static const size_t value = 0; -}; - -template -struct tuple_element { - typedef typename internal::component<__N, typename __T::my_cons>::type type; -}; - -template -inline static typename tuple_element<__N,tuple<__T0,__T1,__T2,__T3,__T4 __TBB_T_PACK > >::type& - get(tuple<__T0,__T1,__T2,__T3,__T4 __TBB_T_PACK >& t) { return internal::get_helper<__N>::get(t); } - -template -inline static typename tuple_element<__N,tuple<__T0,__T1,__T2,__T3,__T4 __TBB_T_PACK > >::type const& - get(const tuple<__T0,__T1,__T2,__T3,__T4 __TBB_T_PACK >& t) { return internal::get_helper<__N>::get(t); } - -} // interface5 -} // tbb - -#if !__TBB_CPP11_TUPLE_PRESENT -namespace tbb { - namespace flow { - using tbb::interface5::tuple; - using tbb::interface5::tuple_size; - using tbb::interface5::tuple_element; - using tbb::interface5::get; - } -} -#endif - -#undef __TBB_T_PACK -#undef __TBB_U_PACK -#undef __TBB_TYPENAME_T_PACK -#undef __TBB_TYPENAME_U_PACK -#undef __TBB_NULL_TYPE_PACK -#undef __TBB_REF_T_PARAM_PACK -#undef __TBB_CONST_REF_T_PARAM_PACK -#undef __TBB_T_PARAM_LIST_PACK -#undef __TBB_CONST_NULL_REF_PACK - -#include "../internal/_warning_suppress_disable_notice.h" -#undef __TBB_tuple_H_include_area - -#endif /* __TBB_tuple_H */ diff --git a/src/3rd party/tbb/concurrent_hash_map.h b/src/3rd party/tbb/concurrent_hash_map.h index e8a617ebda..68652c5961 100644 --- a/src/3rd party/tbb/concurrent_hash_map.h +++ b/src/3rd party/tbb/concurrent_hash_map.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,1637 +14,4 @@ limitations under the License. */ -#ifndef __TBB_concurrent_hash_map_H -#define __TBB_concurrent_hash_map_H - -#define __TBB_concurrent_hash_map_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "tbb_stddef.h" -#include -#include // Need std::pair -#include // Need std::memset -#include __TBB_STD_SWAP_HEADER - -#include "tbb_allocator.h" -#include "spin_rw_mutex.h" -#include "atomic.h" -#include "tbb_exception.h" -#include "tbb_profiling.h" -#include "aligned_space.h" -#include "internal/_tbb_hash_compare_impl.h" -#include "internal/_template_helpers.h" -#include "internal/_allocator_traits.h" -#if __TBB_INITIALIZER_LISTS_PRESENT -#include -#endif -#if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS -#include -#endif -#if __TBB_STATISTICS -#include -#endif -#if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TUPLE_PRESENT -// Definition of __TBB_CPP11_RVALUE_REF_PRESENT includes __TBB_CPP11_TUPLE_PRESENT -// for most of platforms, tuple present macro was added for logical correctness -#include -#endif - -namespace tbb { - -namespace interface5 { - - template, typename A = tbb_allocator > > - class concurrent_hash_map; - - //! @cond INTERNAL - namespace internal { - using namespace tbb::internal; - - - //! Type of a hash code. - typedef size_t hashcode_t; - //! Node base type - struct hash_map_node_base : tbb::internal::no_copy { - //! Mutex type - typedef spin_rw_mutex mutex_t; - //! Scoped lock type for mutex - typedef mutex_t::scoped_lock scoped_t; - //! Next node in chain - hash_map_node_base *next; - mutex_t mutex; - }; - //! Incompleteness flag value - static hash_map_node_base *const rehash_req = reinterpret_cast(size_t(3)); - //! Rehashed empty bucket flag - static hash_map_node_base *const empty_rehashed = reinterpret_cast(size_t(0)); - //! base class of concurrent_hash_map - class hash_map_base { - public: - //! Size type - typedef size_t size_type; - //! Type of a hash code. - typedef size_t hashcode_t; - //! Segment index type - typedef size_t segment_index_t; - //! Node base type - typedef hash_map_node_base node_base; - //! Bucket type - struct bucket : tbb::internal::no_copy { - //! Mutex type for buckets - typedef spin_rw_mutex mutex_t; - //! Scoped lock type for mutex - typedef mutex_t::scoped_lock scoped_t; - mutex_t mutex; - node_base *node_list; - }; - //! Count of segments in the first block - static size_type const embedded_block = 1; - //! Count of segments in the first block - static size_type const embedded_buckets = 1< my_mask; - //! Segment pointers table. Also prevents false sharing between my_mask and my_size - segments_table_t my_table; - //! Size of container in stored items - atomic my_size; // It must be in separate cache line from my_mask due to performance effects - //! Zero segment - bucket my_embedded_segment[embedded_buckets]; -#if __TBB_STATISTICS - atomic my_info_resizes; // concurrent ones - mutable atomic my_info_restarts; // race collisions - atomic my_info_rehashes; // invocations of rehash_bucket -#endif - //! Constructor - hash_map_base() { - std::memset(my_table, 0, sizeof(my_table)); - my_mask = 0; - my_size = 0; - std::memset(my_embedded_segment, 0, sizeof(my_embedded_segment)); - for( size_type i = 0; i < embedded_block; i++ ) // fill the table - my_table[i] = my_embedded_segment + segment_base(i); - my_mask = embedded_buckets - 1; - __TBB_ASSERT( embedded_block <= first_block, "The first block number must include embedded blocks"); -#if __TBB_STATISTICS - my_info_resizes = 0; // concurrent ones - my_info_restarts = 0; // race collisions - my_info_rehashes = 0; // invocations of rehash_bucket -#endif - } - - //! @return segment index of given index in the array - static segment_index_t segment_index_of( size_type index ) { - return segment_index_t( __TBB_Log2( index|1 ) ); - } - - //! @return the first array index of given segment - static segment_index_t segment_base( segment_index_t k ) { - return (segment_index_t(1)<(ptr) > uintptr_t(63); - } - - //! Initialize buckets - static void init_buckets( segment_ptr_t ptr, size_type sz, bool is_initial ) { - if( is_initial ) std::memset( static_cast(ptr), 0, sz*sizeof(bucket) ); - else for(size_type i = 0; i < sz; i++, ptr++) { - *reinterpret_cast(&ptr->mutex) = 0; - ptr->node_list = rehash_req; - } - } - - //! Add node @arg n to bucket @arg b - static void add_to_bucket( bucket *b, node_base *n ) { - __TBB_ASSERT(b->node_list != rehash_req, NULL); - n->next = b->node_list; - b->node_list = n; // its under lock and flag is set - } - - //! Exception safety helper - struct enable_segment_failsafe : tbb::internal::no_copy { - segment_ptr_t *my_segment_ptr; - enable_segment_failsafe(segments_table_t &table, segment_index_t k) : my_segment_ptr(&table[k]) {} - ~enable_segment_failsafe() { - if( my_segment_ptr ) *my_segment_ptr = 0; // indicate no allocation in progress - } - }; - - //! Enable segment - template - void enable_segment( segment_index_t k, const Allocator& allocator, bool is_initial = false ) { - typedef typename tbb::internal::allocator_rebind::type bucket_allocator_type; - typedef tbb::internal::allocator_traits bucket_allocator_traits; - bucket_allocator_type bucket_allocator(allocator); - __TBB_ASSERT( k, "Zero segment must be embedded" ); - enable_segment_failsafe watchdog( my_table, k ); - size_type sz; - __TBB_ASSERT( !is_valid(my_table[k]), "Wrong concurrent assignment"); - if( k >= first_block ) { - sz = segment_size( k ); - segment_ptr_t ptr = bucket_allocator_traits::allocate(bucket_allocator, sz); - init_buckets( ptr, sz, is_initial ); - itt_hide_store_word( my_table[k], ptr ); - sz <<= 1;// double it to get entire capacity of the container - } else { // the first block - __TBB_ASSERT( k == embedded_block, "Wrong segment index" ); - sz = segment_size( first_block ); - segment_ptr_t ptr = bucket_allocator_traits::allocate(bucket_allocator, sz - embedded_buckets); - init_buckets( ptr, sz - embedded_buckets, is_initial ); - ptr -= segment_base(embedded_block); - for(segment_index_t i = embedded_block; i < first_block; i++) // calc the offsets - itt_hide_store_word( my_table[i], ptr + segment_base(i) ); - } - itt_store_word_with_release( my_mask, sz-1 ); - watchdog.my_segment_ptr = 0; - } - - template - void delete_segment(segment_index_t s, const Allocator& allocator) { - typedef typename tbb::internal::allocator_rebind::type bucket_allocator_type; - typedef tbb::internal::allocator_traits bucket_allocator_traits; - bucket_allocator_type bucket_allocator(allocator); - segment_ptr_t buckets_ptr = my_table[s]; - size_type sz = segment_size( s ? s : 1 ); - - if( s >= first_block) // the first segment or the next - bucket_allocator_traits::deallocate(bucket_allocator, buckets_ptr, sz); - else if( s == embedded_block && embedded_block != first_block ) - bucket_allocator_traits::deallocate(bucket_allocator, buckets_ptr, - segment_size(first_block) - embedded_buckets); - if( s >= embedded_block ) my_table[s] = 0; - } - - //! Get bucket by (masked) hashcode - bucket *get_bucket( hashcode_t h ) const throw() { // TODO: add throw() everywhere? - segment_index_t s = segment_index_of( h ); - h -= segment_base(s); - segment_ptr_t seg = my_table[s]; - __TBB_ASSERT( is_valid(seg), "hashcode must be cut by valid mask for allocated segments" ); - return &seg[h]; - } - - // internal serial rehashing helper - void mark_rehashed_levels( hashcode_t h ) throw () { - segment_index_t s = segment_index_of( h ); - while( segment_ptr_t seg = my_table[++s] ) - if( seg[h].node_list == rehash_req ) { - seg[h].node_list = empty_rehashed; - mark_rehashed_levels( h + ((hashcode_t)1<node_list) != rehash_req ) - { -#if __TBB_STATISTICS - my_info_restarts++; // race collisions -#endif - return true; - } - } - return false; - } - - //! Insert a node and check for load factor. @return segment index to enable. - segment_index_t insert_new_node( bucket *b, node_base *n, hashcode_t mask ) { - size_type sz = ++my_size; // prefix form is to enforce allocation after the first item inserted - add_to_bucket( b, n ); - // check load factor - if( sz >= mask ) { // TODO: add custom load_factor - segment_index_t new_seg = __TBB_Log2( mask+1 ); //optimized segment_index_of - __TBB_ASSERT( is_valid(my_table[new_seg-1]), "new allocations must not publish new mask until segment has allocated"); - static const segment_ptr_t is_allocating = (segment_ptr_t)2; - if( !itt_hide_load_word(my_table[new_seg]) - && as_atomic(my_table[new_seg]).compare_and_swap(is_allocating, NULL) == NULL ) - return new_seg; // The value must be processed - } - return 0; - } - - //! Prepare enough segments for number of buckets - template - void reserve(size_type buckets, const Allocator& allocator) { - if( !buckets-- ) return; - bool is_initial = !my_size; - for( size_type m = my_mask; buckets > m; m = my_mask ) - enable_segment( segment_index_of( m+1 ), allocator, is_initial ); - } - //! Swap hash_map_bases - void internal_swap(hash_map_base &table) { - using std::swap; - swap(this->my_mask, table.my_mask); - swap(this->my_size, table.my_size); - for(size_type i = 0; i < embedded_buckets; i++) - swap(this->my_embedded_segment[i].node_list, table.my_embedded_segment[i].node_list); - for(size_type i = embedded_block; i < pointers_per_table; i++) - swap(this->my_table[i], table.my_table[i]); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - void internal_move(hash_map_base&& other) { - my_mask = other.my_mask; - other.my_mask = embedded_buckets - 1; - my_size = other.my_size; - other.my_size = 0; - - for(size_type i = 0; i < embedded_buckets; ++i) { - my_embedded_segment[i].node_list = other.my_embedded_segment[i].node_list; - other.my_embedded_segment[i].node_list = NULL; - } - - for(size_type i = embedded_block; i < pointers_per_table; ++i) { - my_table[i] = other.my_table[i]; - other.my_table[i] = NULL; - } - } -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - }; - - template - class hash_map_range; - - //! Meets requirements of a forward iterator for STL */ - /** Value is either the T or const T type of the container. - @ingroup containers */ - template - class hash_map_iterator - : public std::iterator - { - typedef Container map_type; - typedef typename Container::node node; - typedef hash_map_base::node_base node_base; - typedef hash_map_base::bucket bucket; - - template - friend bool operator==( const hash_map_iterator& i, const hash_map_iterator& j ); - - template - friend bool operator!=( const hash_map_iterator& i, const hash_map_iterator& j ); - - template - friend ptrdiff_t operator-( const hash_map_iterator& i, const hash_map_iterator& j ); - - template - friend class hash_map_iterator; - - template - friend class hash_map_range; - - void advance_to_next_bucket() { // TODO?: refactor to iterator_base class - size_t k = my_index+1; - __TBB_ASSERT( my_bucket, "advancing an invalid iterator?"); - while( k <= my_map->my_mask ) { - // Following test uses 2's-complement wizardry - if( k&(k-2) ) // not the beginning of a segment - ++my_bucket; - else my_bucket = my_map->get_bucket( k ); - my_node = static_cast( my_bucket->node_list ); - if( hash_map_base::is_valid(my_node) ) { - my_index = k; return; - } - ++k; - } - my_bucket = 0; my_node = 0; my_index = k; // the end - } -#if !defined(_MSC_VER) || defined(__INTEL_COMPILER) - template - friend class interface5::concurrent_hash_map; -#else - public: // workaround -#endif - //! concurrent_hash_map over which we are iterating. - const Container *my_map; - - //! Index in hash table for current item - size_t my_index; - - //! Pointer to bucket - const bucket *my_bucket; - - //! Pointer to node that has current item - node *my_node; - - hash_map_iterator( const Container &map, size_t index, const bucket *b, node_base *n ); - - public: - //! Construct undefined iterator - hash_map_iterator(): my_map(), my_index(), my_bucket(), my_node() {} - hash_map_iterator( const hash_map_iterator &other ) : - my_map(other.my_map), - my_index(other.my_index), - my_bucket(other.my_bucket), - my_node(other.my_node) - {} - - hash_map_iterator& operator=( const hash_map_iterator &other ) { - my_map = other.my_map; - my_index = other.my_index; - my_bucket = other.my_bucket; - my_node = other.my_node; - return *this; - } - Value& operator*() const { - __TBB_ASSERT( hash_map_base::is_valid(my_node), "iterator uninitialized or at end of container?" ); - return my_node->value(); - } - Value* operator->() const {return &operator*();} - hash_map_iterator& operator++(); - - //! Post increment - hash_map_iterator operator++(int) { - hash_map_iterator old(*this); - operator++(); - return old; - } - }; - - template - hash_map_iterator::hash_map_iterator( const Container &map, size_t index, const bucket *b, node_base *n ) : - my_map(&map), - my_index(index), - my_bucket(b), - my_node( static_cast(n) ) - { - if( b && !hash_map_base::is_valid(n) ) - advance_to_next_bucket(); - } - - template - hash_map_iterator& hash_map_iterator::operator++() { - my_node = static_cast( my_node->next ); - if( !my_node ) advance_to_next_bucket(); - return *this; - } - - template - bool operator==( const hash_map_iterator& i, const hash_map_iterator& j ) { - return i.my_node == j.my_node && i.my_map == j.my_map; - } - - template - bool operator!=( const hash_map_iterator& i, const hash_map_iterator& j ) { - return i.my_node != j.my_node || i.my_map != j.my_map; - } - - //! Range class used with concurrent_hash_map - /** @ingroup containers */ - template - class hash_map_range { - typedef typename Iterator::map_type map_type; - Iterator my_begin; - Iterator my_end; - mutable Iterator my_midpoint; - size_t my_grainsize; - //! Set my_midpoint to point approximately half way between my_begin and my_end. - void set_midpoint() const; - template friend class hash_map_range; - public: - //! Type for size of a range - typedef std::size_t size_type; - typedef typename Iterator::value_type value_type; - typedef typename Iterator::reference reference; - typedef typename Iterator::difference_type difference_type; - typedef Iterator iterator; - - //! True if range is empty. - bool empty() const {return my_begin==my_end;} - - //! True if range can be partitioned into two subranges. - bool is_divisible() const { - return my_midpoint!=my_end; - } - //! Split range. - hash_map_range( hash_map_range& r, split ) : - my_end(r.my_end), - my_grainsize(r.my_grainsize) - { - r.my_end = my_begin = r.my_midpoint; - __TBB_ASSERT( !empty(), "Splitting despite the range is not divisible" ); - __TBB_ASSERT( !r.empty(), "Splitting despite the range is not divisible" ); - set_midpoint(); - r.set_midpoint(); - } - //! type conversion - template - hash_map_range( hash_map_range& r) : - my_begin(r.my_begin), - my_end(r.my_end), - my_midpoint(r.my_midpoint), - my_grainsize(r.my_grainsize) - {} - //! Init range with container and grainsize specified - hash_map_range( const map_type &map, size_type grainsize_ = 1 ) : - my_begin( Iterator( map, 0, map.my_embedded_segment, map.my_embedded_segment->node_list ) ), - my_end( Iterator( map, map.my_mask + 1, 0, 0 ) ), - my_grainsize( grainsize_ ) - { - __TBB_ASSERT( grainsize_>0, "grainsize must be positive" ); - set_midpoint(); - } - const Iterator& begin() const {return my_begin;} - const Iterator& end() const {return my_end;} - //! The grain size for this range. - size_type grainsize() const {return my_grainsize;} - }; - - template - void hash_map_range::set_midpoint() const { - // Split by groups of nodes - size_t m = my_end.my_index-my_begin.my_index; - if( m > my_grainsize ) { - m = my_begin.my_index + m/2u; - hash_map_base::bucket *b = my_begin.my_map->get_bucket(m); - my_midpoint = Iterator(*my_begin.my_map,m,b,b->node_list); - } else { - my_midpoint = my_end; - } - __TBB_ASSERT( my_begin.my_index <= my_midpoint.my_index, - "my_begin is after my_midpoint" ); - __TBB_ASSERT( my_midpoint.my_index <= my_end.my_index, - "my_midpoint is after my_end" ); - __TBB_ASSERT( my_begin != my_midpoint || my_begin == my_end, - "[my_begin, my_midpoint) range should not be empty" ); - } - - } // internal -//! @endcond - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Suppress "conditional expression is constant" warning. - #pragma warning( push ) - #pragma warning( disable: 4127 ) -#endif - -//! Unordered map from Key to T. -/** concurrent_hash_map is associative container with concurrent access. - -@par Compatibility - The class meets all Container Requirements from C++ Standard (See ISO/IEC 14882:2003(E), clause 23.1). - -@par Exception Safety - - Hash function is not permitted to throw an exception. User-defined types Key and T are forbidden from throwing an exception in destructors. - - If exception happens during insert() operations, it has no effect (unless exception raised by HashCompare::hash() function during grow_segment). - - If exception happens during operator=() operation, the container can have a part of source items, and methods size() and empty() can return wrong results. - -@par Changes since TBB 2.1 - - Replaced internal algorithm and data structure. Patent is pending. - - Added buckets number argument for constructor - -@par Changes since TBB 2.0 - - Fixed exception-safety - - Added template argument for allocator - - Added allocator argument in constructors - - Added constructor from a range of iterators - - Added several new overloaded insert() methods - - Added get_allocator() - - Added swap() - - Added count() - - Added overloaded erase(accessor &) and erase(const_accessor&) - - Added equal_range() [const] - - Added [const_]pointer, [const_]reference, and allocator_type types - - Added global functions: operator==(), operator!=(), and swap() - - @ingroup containers */ -template -class concurrent_hash_map : protected internal::hash_map_base { - template - friend class internal::hash_map_iterator; - - template - friend class internal::hash_map_range; - -public: - typedef Key key_type; - typedef T mapped_type; - typedef std::pair value_type; - typedef hash_map_base::size_type size_type; - typedef ptrdiff_t difference_type; - typedef value_type *pointer; - typedef const value_type *const_pointer; - typedef value_type &reference; - typedef const value_type &const_reference; - typedef internal::hash_map_iterator iterator; - typedef internal::hash_map_iterator const_iterator; - typedef internal::hash_map_range range_type; - typedef internal::hash_map_range const_range_type; - typedef Allocator allocator_type; - -protected: - friend class const_accessor; - class node; - typedef typename tbb::internal::allocator_rebind::type node_allocator_type; - typedef tbb::internal::allocator_traits node_allocator_traits; - node_allocator_type my_allocator; - HashCompare my_hash_compare; - - class node : public node_base { - tbb::aligned_space my_value; - public: - value_type* storage() { return my_value.begin(); } - value_type& value() { return *storage(); } - }; - - void delete_node( node_base *n ) { - node_allocator_traits::destroy(my_allocator, static_cast(n)->storage()); - node_allocator_traits::destroy(my_allocator, static_cast(n)); - node_allocator_traits::deallocate(my_allocator, static_cast(n), 1); - } - - struct node_scoped_guard : tbb::internal::no_copy { - node* my_node; - node_allocator_type& my_alloc; - - node_scoped_guard(node* n, node_allocator_type& alloc) : my_node(n), my_alloc(alloc) {} - ~node_scoped_guard() { - if(my_node) { - node_allocator_traits::destroy(my_alloc, my_node); - node_allocator_traits::deallocate(my_alloc, my_node, 1); - } - } - void dismiss() { my_node = NULL; } - }; - -#if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - static node* create_node(node_allocator_type& allocator, Args&&... args) -#else - template - static node* create_node(node_allocator_type& allocator, __TBB_FORWARDING_REF(Arg1) arg1, __TBB_FORWARDING_REF(Arg2) arg2) -#endif - { - node* node_ptr = node_allocator_traits::allocate(allocator, 1); - node_scoped_guard guard(node_ptr, allocator); - node_allocator_traits::construct(allocator, node_ptr); -#if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - node_allocator_traits::construct(allocator, node_ptr->storage(), std::forward(args)...); -#else - node_allocator_traits::construct(allocator, node_ptr->storage(), tbb::internal::forward(arg1), tbb::internal::forward(arg2)); -#endif - guard.dismiss(); - return node_ptr; - } - - static node* allocate_node_copy_construct(node_allocator_type& allocator, const Key &key, const T * t){ - return create_node(allocator, key, *t); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - static node* allocate_node_move_construct(node_allocator_type& allocator, const Key &key, const T * t){ - return create_node(allocator, key, std::move(*const_cast(t))); - } -#endif - - static node* allocate_node_default_construct(node_allocator_type& allocator, const Key &key, const T * ){ -#if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TUPLE_PRESENT - // Emplace construct an empty T object inside the pair - return create_node(allocator, std::piecewise_construct, - std::forward_as_tuple(key), std::forward_as_tuple()); -#else - // Use of a temporary object is impossible, because create_node takes a non-const reference. - // copy-initialization is possible because T is already required to be CopyConstructible. - T obj = T(); - return create_node(allocator, key, tbb::internal::move(obj)); -#endif - } - - static node* do_not_allocate_node(node_allocator_type& , const Key &, const T * ){ - __TBB_ASSERT(false,"this dummy function should not be called"); - return NULL; - } - - node *search_bucket( const key_type &key, bucket *b ) const { - node *n = static_cast( b->node_list ); - while( is_valid(n) && !my_hash_compare.equal(key, n->value().first) ) - n = static_cast( n->next ); - __TBB_ASSERT(n != internal::rehash_req, "Search can be executed only for rehashed bucket"); - return n; - } - - //! bucket accessor is to find, rehash, acquire a lock, and access a bucket - class bucket_accessor : public bucket::scoped_t { - bucket *my_b; - public: - bucket_accessor( concurrent_hash_map *base, const hashcode_t h, bool writer = false ) { acquire( base, h, writer ); } - //! find a bucket by masked hashcode, optionally rehash, and acquire the lock - inline void acquire( concurrent_hash_map *base, const hashcode_t h, bool writer = false ) { - my_b = base->get_bucket( h ); - // TODO: actually, notification is unnecessary here, just hiding double-check - if( itt_load_word_with_acquire(my_b->node_list) == internal::rehash_req - && try_acquire( my_b->mutex, /*write=*/true ) ) - { - if( my_b->node_list == internal::rehash_req ) base->rehash_bucket( my_b, h ); //recursive rehashing - } - else bucket::scoped_t::acquire( my_b->mutex, writer ); - __TBB_ASSERT( my_b->node_list != internal::rehash_req, NULL); - } - //! check whether bucket is locked for write - bool is_writer() { return bucket::scoped_t::is_writer; } - //! get bucket pointer - bucket *operator() () { return my_b; } - }; - - // TODO refactor to hash_base - void rehash_bucket( bucket *b_new, const hashcode_t h ) { - __TBB_ASSERT( *(intptr_t*)(&b_new->mutex), "b_new must be locked (for write)"); - __TBB_ASSERT( h > 1, "The lowermost buckets can't be rehashed" ); - __TBB_store_with_release(b_new->node_list, internal::empty_rehashed); // mark rehashed - hashcode_t mask = ( 1u<<__TBB_Log2( h ) ) - 1; // get parent mask from the topmost bit -#if __TBB_STATISTICS - my_info_rehashes++; // invocations of rehash_bucket -#endif - - bucket_accessor b_old( this, h & mask ); - - mask = (mask<<1) | 1; // get full mask for new bucket - __TBB_ASSERT( (mask&(mask+1))==0 && (h & mask) == h, NULL ); - restart: - for( node_base **p = &b_old()->node_list, *n = __TBB_load_with_acquire(*p); is_valid(n); n = *p ) { - hashcode_t c = my_hash_compare.hash( static_cast(n)->value().first ); -#if TBB_USE_ASSERT - hashcode_t bmask = h & (mask>>1); - bmask = bmask==0? 1 : ( 1u<<(__TBB_Log2( bmask )+1 ) ) - 1; // minimal mask of parent bucket - __TBB_ASSERT( (c & bmask) == (h & bmask), "hash() function changed for key in table" ); -#endif - if( (c & mask) == h ) { - if( !b_old.is_writer() ) - if( !b_old.upgrade_to_writer() ) { - goto restart; // node ptr can be invalid due to concurrent erase - } - *p = n->next; // exclude from b_old - add_to_bucket( b_new, n ); - } else p = &n->next; // iterate to next item - } - } - - struct call_clear_on_leave { - concurrent_hash_map* my_ch_map; - call_clear_on_leave( concurrent_hash_map* a_ch_map ) : my_ch_map(a_ch_map) {} - void dismiss() {my_ch_map = 0;} - ~call_clear_on_leave(){ - if (my_ch_map){ - my_ch_map->clear(); - } - } - }; -public: - - class accessor; - //! Combines data access, locking, and garbage collection. - class const_accessor : private node::scoped_t /*which derived from no_copy*/ { - friend class concurrent_hash_map; - friend class accessor; - public: - //! Type of value - typedef const typename concurrent_hash_map::value_type value_type; - - //! True if result is empty. - bool empty() const { return !my_node; } - - //! Set to null - void release() { - if( my_node ) { - node::scoped_t::release(); - my_node = 0; - } - } - - //! Return reference to associated value in hash table. - const_reference operator*() const { - __TBB_ASSERT( my_node, "attempt to dereference empty accessor" ); - return my_node->value(); - } - - //! Return pointer to associated value in hash table. - const_pointer operator->() const { - return &operator*(); - } - - //! Create empty result - const_accessor() : my_node(NULL) {} - - //! Destroy result after releasing the underlying reference. - ~const_accessor() { - my_node = NULL; // scoped lock's release() is called in its destructor - } - protected: - bool is_writer() { return node::scoped_t::is_writer; } - node *my_node; - hashcode_t my_hash; - }; - - //! Allows write access to elements and combines data access, locking, and garbage collection. - class accessor: public const_accessor { - public: - //! Type of value - typedef typename concurrent_hash_map::value_type value_type; - - //! Return reference to associated value in hash table. - reference operator*() const { - __TBB_ASSERT( this->my_node, "attempt to dereference empty accessor" ); - return this->my_node->value(); - } - - //! Return pointer to associated value in hash table. - pointer operator->() const { - return &operator*(); - } - }; - - //! Construct empty table. - explicit concurrent_hash_map( const allocator_type &a = allocator_type() ) - : internal::hash_map_base(), my_allocator(a) - {} - - explicit concurrent_hash_map( const HashCompare& compare, const allocator_type& a = allocator_type() ) - : internal::hash_map_base(), my_allocator(a), my_hash_compare(compare) - {} - - //! Construct empty table with n preallocated buckets. This number serves also as initial concurrency level. - concurrent_hash_map( size_type n, const allocator_type &a = allocator_type() ) - : internal::hash_map_base(), my_allocator(a) - { - reserve( n, my_allocator ); - } - - concurrent_hash_map( size_type n, const HashCompare& compare, const allocator_type& a = allocator_type() ) - : internal::hash_map_base(), my_allocator(a), my_hash_compare(compare) - { - reserve( n, my_allocator ); - } - - //! Copy constructor - concurrent_hash_map( const concurrent_hash_map &table ) - : internal::hash_map_base(), - my_allocator(node_allocator_traits::select_on_container_copy_construction(table.get_allocator())) - { - call_clear_on_leave scope_guard(this); - internal_copy(table); - scope_guard.dismiss(); - } - - concurrent_hash_map( const concurrent_hash_map &table, const allocator_type &a) - : internal::hash_map_base(), my_allocator(a) - { - call_clear_on_leave scope_guard(this); - internal_copy(table); - scope_guard.dismiss(); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructor - concurrent_hash_map( concurrent_hash_map &&table ) - : internal::hash_map_base(), my_allocator(std::move(table.get_allocator())) - { - internal_move(std::move(table)); - } - - //! Move constructor - concurrent_hash_map( concurrent_hash_map &&table, const allocator_type &a ) - : internal::hash_map_base(), my_allocator(a) - { - if (a == table.get_allocator()){ - internal_move(std::move(table)); - }else{ - call_clear_on_leave scope_guard(this); - internal_copy(std::make_move_iterator(table.begin()), std::make_move_iterator(table.end()), table.size()); - scope_guard.dismiss(); - } - } -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - //! Construction with copying iteration range and given allocator instance - template - concurrent_hash_map( I first, I last, const allocator_type &a = allocator_type() ) - : internal::hash_map_base(), my_allocator(a) - { - call_clear_on_leave scope_guard(this); - internal_copy(first, last, std::distance(first, last)); - scope_guard.dismiss(); - } - - template - concurrent_hash_map( I first, I last, const HashCompare& compare, const allocator_type& a = allocator_type() ) - : internal::hash_map_base(), my_allocator(a), my_hash_compare(compare) - { - call_clear_on_leave scope_guard(this); - internal_copy(first, last, std::distance(first, last)); - scope_guard.dismiss(); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Construct empty table with n preallocated buckets. This number serves also as initial concurrency level. - concurrent_hash_map( std::initializer_list il, const allocator_type &a = allocator_type() ) - : internal::hash_map_base(), my_allocator(a) - { - call_clear_on_leave scope_guard(this); - internal_copy(il.begin(), il.end(), il.size()); - scope_guard.dismiss(); - } - - concurrent_hash_map( std::initializer_list il, const HashCompare& compare, const allocator_type& a = allocator_type() ) - : internal::hash_map_base(), my_allocator(a), my_hash_compare(compare) - { - call_clear_on_leave scope_guard(this); - internal_copy(il.begin(), il.end(), il.size()); - scope_guard.dismiss(); - } - -#endif //__TBB_INITIALIZER_LISTS_PRESENT - - //! Assignment - concurrent_hash_map& operator=( const concurrent_hash_map &table ) { - if( this!=&table ) { - typedef typename node_allocator_traits::propagate_on_container_copy_assignment pocca_type; - clear(); - tbb::internal::allocator_copy_assignment(my_allocator, table.my_allocator, pocca_type()); - internal_copy(table); - } - return *this; - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move Assignment - concurrent_hash_map& operator=( concurrent_hash_map &&table ) { - if(this != &table) { - typedef typename node_allocator_traits::propagate_on_container_move_assignment pocma_type; - internal_move_assign(std::move(table), pocma_type()); - } - return *this; - } -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Assignment - concurrent_hash_map& operator=( std::initializer_list il ) { - clear(); - internal_copy(il.begin(), il.end(), il.size()); - return *this; - } -#endif //__TBB_INITIALIZER_LISTS_PRESENT - - - //! Rehashes and optionally resizes the whole table. - /** Useful to optimize performance before or after concurrent operations. - Also enables using of find() and count() concurrent methods in serial context. */ - void rehash(size_type n = 0); - - //! Clear table - void clear(); - - //! Clear table and destroy it. - ~concurrent_hash_map() { clear(); } - - //------------------------------------------------------------------------ - // Parallel algorithm support - //------------------------------------------------------------------------ - range_type range( size_type grainsize=1 ) { - return range_type( *this, grainsize ); - } - const_range_type range( size_type grainsize=1 ) const { - return const_range_type( *this, grainsize ); - } - - //------------------------------------------------------------------------ - // STL support - not thread-safe methods - //------------------------------------------------------------------------ - iterator begin() { return iterator( *this, 0, my_embedded_segment, my_embedded_segment->node_list ); } - iterator end() { return iterator( *this, 0, 0, 0 ); } - const_iterator begin() const { return const_iterator( *this, 0, my_embedded_segment, my_embedded_segment->node_list ); } - const_iterator end() const { return const_iterator( *this, 0, 0, 0 ); } - std::pair equal_range( const Key& key ) { return internal_equal_range( key, end() ); } - std::pair equal_range( const Key& key ) const { return internal_equal_range( key, end() ); } - - //! Number of items in table. - size_type size() const { return my_size; } - - //! True if size()==0. - bool empty() const { return my_size == 0; } - - //! Upper bound on size. - size_type max_size() const {return (~size_type(0))/sizeof(node);} - - //! Returns the current number of buckets - size_type bucket_count() const { return my_mask+1; } - - //! return allocator object - allocator_type get_allocator() const { return this->my_allocator; } - - //! swap two instances. Iterators are invalidated - void swap( concurrent_hash_map &table ); - - //------------------------------------------------------------------------ - // concurrent map operations - //------------------------------------------------------------------------ - - //! Return count of items (0 or 1) - size_type count( const Key &key ) const { - return const_cast(this)->lookup(/*insert*/false, key, NULL, NULL, /*write=*/false, &do_not_allocate_node ); - } - - //! Find item and acquire a read lock on the item. - /** Return true if item is found, false otherwise. */ - bool find( const_accessor &result, const Key &key ) const { - result.release(); - return const_cast(this)->lookup(/*insert*/false, key, NULL, &result, /*write=*/false, &do_not_allocate_node ); - } - - //! Find item and acquire a write lock on the item. - /** Return true if item is found, false otherwise. */ - bool find( accessor &result, const Key &key ) { - result.release(); - return lookup(/*insert*/false, key, NULL, &result, /*write=*/true, &do_not_allocate_node ); - } - - //! Insert item (if not already present) and acquire a read lock on the item. - /** Returns true if item is new. */ - bool insert( const_accessor &result, const Key &key ) { - result.release(); - return lookup(/*insert*/true, key, NULL, &result, /*write=*/false, &allocate_node_default_construct ); - } - - //! Insert item (if not already present) and acquire a write lock on the item. - /** Returns true if item is new. */ - bool insert( accessor &result, const Key &key ) { - result.release(); - return lookup(/*insert*/true, key, NULL, &result, /*write=*/true, &allocate_node_default_construct ); - } - - //! Insert item by copying if there is no such key present already and acquire a read lock on the item. - /** Returns true if item is new. */ - bool insert( const_accessor &result, const value_type &value ) { - result.release(); - return lookup(/*insert*/true, value.first, &value.second, &result, /*write=*/false, &allocate_node_copy_construct ); - } - - //! Insert item by copying if there is no such key present already and acquire a write lock on the item. - /** Returns true if item is new. */ - bool insert( accessor &result, const value_type &value ) { - result.release(); - return lookup(/*insert*/true, value.first, &value.second, &result, /*write=*/true, &allocate_node_copy_construct ); - } - - //! Insert item by copying if there is no such key present already - /** Returns true if item is inserted. */ - bool insert( const value_type &value ) { - return lookup(/*insert*/true, value.first, &value.second, NULL, /*write=*/false, &allocate_node_copy_construct ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Insert item by copying if there is no such key present already and acquire a read lock on the item. - /** Returns true if item is new. */ - bool insert( const_accessor &result, value_type && value ) { - return generic_move_insert(result, std::move(value)); - } - - //! Insert item by copying if there is no such key present already and acquire a write lock on the item. - /** Returns true if item is new. */ - bool insert( accessor &result, value_type && value ) { - return generic_move_insert(result, std::move(value)); - } - - //! Insert item by copying if there is no such key present already - /** Returns true if item is inserted. */ - bool insert( value_type && value ) { - return generic_move_insert(accessor_not_used(), std::move(value)); - } - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - //! Insert item by copying if there is no such key present already and acquire a read lock on the item. - /** Returns true if item is new. */ - template - bool emplace( const_accessor &result, Args&&... args ) { - return generic_emplace(result, std::forward(args)...); - } - - //! Insert item by copying if there is no such key present already and acquire a write lock on the item. - /** Returns true if item is new. */ - template - bool emplace( accessor &result, Args&&... args ) { - return generic_emplace(result, std::forward(args)...); - } - - //! Insert item by copying if there is no such key present already - /** Returns true if item is inserted. */ - template - bool emplace( Args&&... args ) { - return generic_emplace(accessor_not_used(), std::forward(args)...); - } -#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - //! Insert range [first, last) - template - void insert( I first, I last ) { - for ( ; first != last; ++first ) - insert( *first ); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Insert initializer list - void insert( std::initializer_list il ) { - insert( il.begin(), il.end() ); - } -#endif //__TBB_INITIALIZER_LISTS_PRESENT - - //! Erase item. - /** Return true if item was erased by particularly this call. */ - bool erase( const Key& key ); - - //! Erase item by const_accessor. - /** Return true if item was erased by particularly this call. */ - bool erase( const_accessor& item_accessor ) { - return exclude( item_accessor ); - } - - //! Erase item by accessor. - /** Return true if item was erased by particularly this call. */ - bool erase( accessor& item_accessor ) { - return exclude( item_accessor ); - } - -protected: - //! Insert or find item and optionally acquire a lock on the item. - bool lookup(bool op_insert, const Key &key, const T *t, const_accessor *result, bool write, node* (*allocate_node)(node_allocator_type& , const Key &, const T * ), node *tmp_n = 0 ) ; - - struct accessor_not_used { void release(){}}; - friend const_accessor* accessor_location( accessor_not_used const& ){ return NULL;} - friend const_accessor* accessor_location( const_accessor & a ) { return &a;} - - friend bool is_write_access_needed( accessor const& ) { return true;} - friend bool is_write_access_needed( const_accessor const& ) { return false;} - friend bool is_write_access_needed( accessor_not_used const& ) { return false;} - -#if __TBB_CPP11_RVALUE_REF_PRESENT - template - bool generic_move_insert( Accessor && result, value_type && value ) { - result.release(); - return lookup(/*insert*/true, value.first, &value.second, accessor_location(result), is_write_access_needed(result), &allocate_node_move_construct ); - } - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - bool generic_emplace( Accessor && result, Args &&... args ) { - result.release(); - node * node_ptr = create_node(my_allocator, std::forward(args)...); - return lookup(/*insert*/true, node_ptr->value().first, NULL, accessor_location(result), is_write_access_needed(result), &do_not_allocate_node, node_ptr ); - } -#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - //! delete item by accessor - bool exclude( const_accessor &item_accessor ); - - //! Returns an iterator for an item defined by the key, or for the next item after it (if upper==true) - template - std::pair internal_equal_range( const Key& key, I end ) const; - - //! Copy "source" to *this, where *this must start out empty. - void internal_copy( const concurrent_hash_map& source ); - - template - void internal_copy( I first, I last, size_type reserve_size ); - -#if __TBB_CPP11_RVALUE_REF_PRESENT - // A compile-time dispatch to allow move assignment of containers with non-movable value_type if POCMA is true_type - void internal_move_assign(concurrent_hash_map&& other, tbb::internal::traits_true_type) { - tbb::internal::allocator_move_assignment(my_allocator, other.my_allocator, tbb::internal::traits_true_type()); - internal_move(std::move(other)); - } - - void internal_move_assign(concurrent_hash_map&& other, tbb::internal::traits_false_type) { - if (this->my_allocator == other.my_allocator) { - internal_move(std::move(other)); - } else { - //do per element move - internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end()), other.size()); - } - } -#endif - - //! Fast find when no concurrent erasure is used. For internal use inside TBB only! - /** Return pointer to item with given key, or NULL if no such item exists. - Must not be called concurrently with erasure operations. */ - const_pointer internal_fast_find( const Key& key ) const { - hashcode_t h = my_hash_compare.hash( key ); - hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask ); - node *n; - restart: - __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); - bucket *b = get_bucket( h & m ); - // TODO: actually, notification is unnecessary here, just hiding double-check - if( itt_load_word_with_acquire(b->node_list) == internal::rehash_req ) - { - bucket::scoped_t lock; - if( lock.try_acquire( b->mutex, /*write=*/true ) ) { - if( b->node_list == internal::rehash_req) - const_cast(this)->rehash_bucket( b, h & m ); //recursive rehashing - } - else lock.acquire( b->mutex, /*write=*/false ); - __TBB_ASSERT(b->node_list!=internal::rehash_req,NULL); - } - n = search_bucket( key, b ); - if( n ) - return &n->item; - else if( check_mask_race( h, m ) ) - goto restart; - return 0; - } -}; - -#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT -namespace internal { -using namespace tbb::internal; - -template typename Map, typename Key, typename T, typename... Args> -using hash_map_t = Map< - Key, T, - std::conditional_t< (sizeof...(Args)>0) && !is_allocator_v< pack_element_t<0, Args...> >, - pack_element_t<0, Args...>, tbb_hash_compare >, - std::conditional_t< (sizeof...(Args)>0) && is_allocator_v< pack_element_t >, - pack_element_t, tbb_allocator > > ->; -} - -// Deduction guide for the constructor from two iterators and hash_compare/ allocator -template -concurrent_hash_map(I, I, Args...) --> internal::hash_map_t,internal::iterator_mapped_t, Args...>; - -// Deduction guide for the constructor from an initializer_list and hash_compare/ allocator -// Deduction guide for an initializer_list, hash_compare and allocator is implicit -template -concurrent_hash_map(std::initializer_list>, CompareOrAllocator) --> internal::hash_map_t; - -#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ - -template -bool concurrent_hash_map::lookup( bool op_insert, const Key &key, const T *t, const_accessor *result, bool write, node* (*allocate_node)(node_allocator_type& , const Key&, const T*), node *tmp_n ) { - __TBB_ASSERT( !result || !result->my_node, NULL ); - bool return_value; - hashcode_t const h = my_hash_compare.hash( key ); - hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask ); - segment_index_t grow_segment = 0; - node *n; - restart: - {//lock scope - __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); - return_value = false; - // get bucket - bucket_accessor b( this, h & m ); - - // find a node - n = search_bucket( key, b() ); - if( op_insert ) { - // [opt] insert a key - if( !n ) { - if( !tmp_n ) { - tmp_n = allocate_node(my_allocator, key, t); - } - if( !b.is_writer() && !b.upgrade_to_writer() ) { // TODO: improved insertion - // Rerun search_list, in case another thread inserted the item during the upgrade. - n = search_bucket( key, b() ); - if( is_valid(n) ) { // unfortunately, it did - b.downgrade_to_reader(); - goto exists; - } - } - if( check_mask_race(h, m) ) - goto restart; // b.release() is done in ~b(). - // insert and set flag to grow the container - grow_segment = insert_new_node( b(), n = tmp_n, m ); - tmp_n = 0; - return_value = true; - } - } else { // find or count - if( !n ) { - if( check_mask_race( h, m ) ) - goto restart; // b.release() is done in ~b(). TODO: replace by continue - return false; - } - return_value = true; - } - exists: - if( !result ) goto check_growth; - // TODO: the following seems as generic/regular operation - // acquire the item - if( !result->try_acquire( n->mutex, write ) ) { - for( tbb::internal::atomic_backoff backoff(true);; ) { - if( result->try_acquire( n->mutex, write ) ) break; - if( !backoff.bounded_pause() ) { - // the wait takes really long, restart the operation - b.release(); - __TBB_ASSERT( !op_insert || !return_value, "Can't acquire new item in locked bucket?" ); - __TBB_Yield(); - m = (hashcode_t) itt_load_word_with_acquire( my_mask ); - goto restart; - } - } - } - }//lock scope - result->my_node = n; - result->my_hash = h; -check_growth: - // [opt] grow the container - if( grow_segment ) { -#if __TBB_STATISTICS - my_info_resizes++; // concurrent ones -#endif - enable_segment( grow_segment, my_allocator ); - } - if( tmp_n ) // if op_insert only - delete_node( tmp_n ); - return return_value; -} - -template -template -std::pair concurrent_hash_map::internal_equal_range( const Key& key, I end_ ) const { - hashcode_t h = my_hash_compare.hash( key ); - hashcode_t m = my_mask; - __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); - h &= m; - bucket *b = get_bucket( h ); - while( b->node_list == internal::rehash_req ) { - m = ( 1u<<__TBB_Log2( h ) ) - 1; // get parent mask from the topmost bit - b = get_bucket( h &= m ); - } - node *n = search_bucket( key, b ); - if( !n ) - return std::make_pair(end_, end_); - iterator lower(*this, h, b, n), upper(lower); - return std::make_pair(lower, ++upper); -} - -template -bool concurrent_hash_map::exclude( const_accessor &item_accessor ) { - __TBB_ASSERT( item_accessor.my_node, NULL ); - node_base *const n = item_accessor.my_node; - hashcode_t const h = item_accessor.my_hash; - hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask ); - do { - // get bucket - bucket_accessor b( this, h & m, /*writer=*/true ); - node_base **p = &b()->node_list; - while( *p && *p != n ) - p = &(*p)->next; - if( !*p ) { // someone else was first - if( check_mask_race( h, m ) ) - continue; - item_accessor.release(); - return false; - } - __TBB_ASSERT( *p == n, NULL ); - *p = n->next; // remove from container - my_size--; - break; - } while(true); - if( !item_accessor.is_writer() ) // need to get exclusive lock - item_accessor.upgrade_to_writer(); // return value means nothing here - item_accessor.release(); - delete_node( n ); // Only one thread can delete it - return true; -} - -template -bool concurrent_hash_map::erase( const Key &key ) { - node_base *n; - hashcode_t const h = my_hash_compare.hash( key ); - hashcode_t m = (hashcode_t) itt_load_word_with_acquire( my_mask ); -restart: - {//lock scope - // get bucket - bucket_accessor b( this, h & m ); - search: - node_base **p = &b()->node_list; - n = *p; - while( is_valid(n) && !my_hash_compare.equal(key, static_cast(n)->value().first ) ) { - p = &n->next; - n = *p; - } - if( !n ) { // not found, but mask could be changed - if( check_mask_race( h, m ) ) - goto restart; - return false; - } - else if( !b.is_writer() && !b.upgrade_to_writer() ) { - if( check_mask_race( h, m ) ) // contended upgrade, check mask - goto restart; - goto search; - } - *p = n->next; - my_size--; - } - { - typename node::scoped_t item_locker( n->mutex, /*write=*/true ); - } - // note: there should be no threads pretending to acquire this mutex again, do not try to upgrade const_accessor! - delete_node( n ); // Only one thread can delete it due to write lock on the bucket - return true; -} - -template -void concurrent_hash_map::swap(concurrent_hash_map &table) { - typedef typename node_allocator_traits::propagate_on_container_swap pocs_type; - if (this != &table && (pocs_type::value || my_allocator == table.my_allocator)) { - using std::swap; - tbb::internal::allocator_swap(this->my_allocator, table.my_allocator, pocs_type()); - swap(this->my_hash_compare, table.my_hash_compare); - internal_swap(table); - } -} - -template -void concurrent_hash_map::rehash(size_type sz) { - reserve( sz, my_allocator ); // TODO: add reduction of number of buckets as well - hashcode_t mask = my_mask; - hashcode_t b = (mask+1)>>1; // size or first index of the last segment - __TBB_ASSERT((b&(b-1))==0, NULL); // zero or power of 2 - bucket *bp = get_bucket( b ); // only the last segment should be scanned for rehashing - for(; b <= mask; b++, bp++ ) { - node_base *n = bp->node_list; - __TBB_ASSERT( is_valid(n) || n == internal::empty_rehashed || n == internal::rehash_req, "Broken internal structure" ); - __TBB_ASSERT( *reinterpret_cast(&bp->mutex) == 0, "concurrent or unexpectedly terminated operation during rehash() execution" ); - if( n == internal::rehash_req ) { // rehash bucket, conditional because rehashing of a previous bucket may affect this one - hashcode_t h = b; bucket *b_old = bp; - do { - __TBB_ASSERT( h > 1, "The lowermost buckets can't be rehashed" ); - hashcode_t m = ( 1u<<__TBB_Log2( h ) ) - 1; // get parent mask from the topmost bit - b_old = get_bucket( h &= m ); - } while( b_old->node_list == internal::rehash_req ); - // now h - is index of the root rehashed bucket b_old - mark_rehashed_levels( h ); // mark all non-rehashed children recursively across all segments - for( node_base **p = &b_old->node_list, *q = *p; is_valid(q); q = *p ) { - hashcode_t c = my_hash_compare.hash( static_cast(q)->value().first ); - if( (c & mask) != h ) { // should be rehashed - *p = q->next; // exclude from b_old - bucket *b_new = get_bucket( c & mask ); - __TBB_ASSERT( b_new->node_list != internal::rehash_req, "hash() function changed for key in table or internal error" ); - add_to_bucket( b_new, q ); - } else p = &q->next; // iterate to next item - } - } - } -#if TBB_USE_PERFORMANCE_WARNINGS - int current_size = int(my_size), buckets = int(mask)+1, empty_buckets = 0, overpopulated_buckets = 0; // usage statistics - static bool reported = false; -#endif -#if TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS - for( b = 0; b <= mask; b++ ) {// only last segment should be scanned for rehashing - if( b & (b-2) ) ++bp; // not the beginning of a segment - else bp = get_bucket( b ); - node_base *n = bp->node_list; - __TBB_ASSERT( *reinterpret_cast(&bp->mutex) == 0, "concurrent or unexpectedly terminated operation during rehash() execution" ); - __TBB_ASSERT( is_valid(n) || n == internal::empty_rehashed, "Broken internal structure" ); -#if TBB_USE_PERFORMANCE_WARNINGS - if( n == internal::empty_rehashed ) empty_buckets++; - else if( n->next ) overpopulated_buckets++; -#endif -#if TBB_USE_ASSERT - for( ; is_valid(n); n = n->next ) { - hashcode_t h = my_hash_compare.hash( static_cast(n)->value().first ) & mask; - __TBB_ASSERT( h == b, "hash() function changed for key in table or internal error" ); - } -#endif - } -#endif // TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS -#if TBB_USE_PERFORMANCE_WARNINGS - if( buckets > current_size) empty_buckets -= buckets - current_size; - else overpopulated_buckets -= current_size - buckets; // TODO: load_factor? - if( !reported && buckets >= 512 && ( 2*empty_buckets > current_size || 2*overpopulated_buckets > current_size ) ) { - tbb::internal::runtime_warning( - "Performance is not optimal because the hash function produces bad randomness in lower bits in %s.\nSize: %d Empties: %d Overlaps: %d", -#if __TBB_USE_OPTIONAL_RTTI - typeid(*this).name(), -#else - "concurrent_hash_map", -#endif - current_size, empty_buckets, overpopulated_buckets ); - reported = true; - } -#endif -} - -template -void concurrent_hash_map::clear() { - hashcode_t m = my_mask; - __TBB_ASSERT((m&(m+1))==0, "data structure is invalid"); -#if TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS -#if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS - int current_size = int(my_size), buckets = int(m)+1, empty_buckets = 0, overpopulated_buckets = 0; // usage statistics - static bool reported = false; -#endif - bucket *bp = 0; - // check consistency - for( segment_index_t b = 0; b <= m; b++ ) { - if( b & (b-2) ) ++bp; // not the beginning of a segment - else bp = get_bucket( b ); - node_base *n = bp->node_list; - __TBB_ASSERT( is_valid(n) || n == internal::empty_rehashed || n == internal::rehash_req, "Broken internal structure" ); - __TBB_ASSERT( *reinterpret_cast(&bp->mutex) == 0, "concurrent or unexpectedly terminated operation during clear() execution" ); -#if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS - if( n == internal::empty_rehashed ) empty_buckets++; - else if( n == internal::rehash_req ) buckets--; - else if( n->next ) overpopulated_buckets++; -#endif -#if __TBB_EXTRA_DEBUG - for(; is_valid(n); n = n->next ) { - hashcode_t h = my_hash_compare.hash( static_cast(n)->value().first ); - h &= m; - __TBB_ASSERT( h == b || get_bucket(h)->node_list == internal::rehash_req, "hash() function changed for key in table or internal error" ); - } -#endif - } -#if TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS -#if __TBB_STATISTICS - printf( "items=%d buckets: capacity=%d rehashed=%d empty=%d overpopulated=%d" - " concurrent: resizes=%u rehashes=%u restarts=%u\n", - current_size, int(m+1), buckets, empty_buckets, overpopulated_buckets, - unsigned(my_info_resizes), unsigned(my_info_rehashes), unsigned(my_info_restarts) ); - my_info_resizes = 0; // concurrent ones - my_info_restarts = 0; // race collisions - my_info_rehashes = 0; // invocations of rehash_bucket -#endif - if( buckets > current_size) empty_buckets -= buckets - current_size; - else overpopulated_buckets -= current_size - buckets; // TODO: load_factor? - if( !reported && buckets >= 512 && ( 2*empty_buckets > current_size || 2*overpopulated_buckets > current_size ) ) { - tbb::internal::runtime_warning( - "Performance is not optimal because the hash function produces bad randomness in lower bits in %s.\nSize: %d Empties: %d Overlaps: %d", -#if __TBB_USE_OPTIONAL_RTTI - typeid(*this).name(), -#else - "concurrent_hash_map", -#endif - current_size, empty_buckets, overpopulated_buckets ); - reported = true; - } -#endif -#endif // TBB_USE_ASSERT || TBB_USE_PERFORMANCE_WARNINGS || __TBB_STATISTICS - my_size = 0; - segment_index_t s = segment_index_of( m ); - __TBB_ASSERT( s+1 == pointers_per_table || !my_table[s+1], "wrong mask or concurrent grow" ); - do { - __TBB_ASSERT( is_valid( my_table[s] ), "wrong mask or concurrent grow" ); - segment_ptr_t buckets_ptr = my_table[s]; - size_type sz = segment_size( s ? s : 1 ); - for( segment_index_t i = 0; i < sz; i++ ) - for( node_base *n = buckets_ptr[i].node_list; is_valid(n); n = buckets_ptr[i].node_list ) { - buckets_ptr[i].node_list = n->next; - delete_node( n ); - } - delete_segment(s, my_allocator); - } while(s-- > 0); - my_mask = embedded_buckets - 1; -} - -template -void concurrent_hash_map::internal_copy( const concurrent_hash_map& source ) { - hashcode_t mask = source.my_mask; - if( my_mask == mask ) { // optimized version - reserve( source.my_size, my_allocator ); // TODO: load_factor? - bucket *dst = 0, *src = 0; - bool rehash_required = false; - for( hashcode_t k = 0; k <= mask; k++ ) { - if( k & (k-2) ) ++dst,src++; // not the beginning of a segment - else { dst = get_bucket( k ); src = source.get_bucket( k ); } - __TBB_ASSERT( dst->node_list != internal::rehash_req, "Invalid bucket in destination table"); - node *n = static_cast( src->node_list ); - if( n == internal::rehash_req ) { // source is not rehashed, items are in previous buckets - rehash_required = true; - dst->node_list = internal::rehash_req; - } else for(; n; n = static_cast( n->next ) ) { - node* node_ptr = create_node(my_allocator, n->value().first, n->value().second); - add_to_bucket( dst, node_ptr); - ++my_size; // TODO: replace by non-atomic op - } - } - if( rehash_required ) rehash(); - } else internal_copy( source.begin(), source.end(), source.my_size ); -} - -template -template -void concurrent_hash_map::internal_copy(I first, I last, size_type reserve_size) { - reserve( reserve_size, my_allocator ); // TODO: load_factor? - hashcode_t m = my_mask; - for(; first != last; ++first) { - hashcode_t h = my_hash_compare.hash( (*first).first ); - bucket *b = get_bucket( h & m ); - __TBB_ASSERT( b->node_list != internal::rehash_req, "Invalid bucket in destination table"); - node* node_ptr = create_node(my_allocator, (*first).first, (*first).second); - add_to_bucket( b, node_ptr ); - ++my_size; // TODO: replace by non-atomic op - } -} - -} // namespace interface5 - -using interface5::concurrent_hash_map; - - -template -inline bool operator==(const concurrent_hash_map &a, const concurrent_hash_map &b) { - if(a.size() != b.size()) return false; - typename concurrent_hash_map::const_iterator i(a.begin()), i_end(a.end()); - typename concurrent_hash_map::const_iterator j, j_end(b.end()); - for(; i != i_end; ++i) { - j = b.equal_range(i->first).first; - if( j == j_end || !(i->second == j->second) ) return false; - } - return true; -} - -template -inline bool operator!=(const concurrent_hash_map &a, const concurrent_hash_map &b) -{ return !(a == b); } - -template -inline void swap(concurrent_hash_map &a, concurrent_hash_map &b) -{ a.swap( b ); } - -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning( pop ) -#endif // warning 4127 is back - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_concurrent_hash_map_H_include_area - -#endif /* __TBB_concurrent_hash_map_H */ +#include "../oneapi/tbb/concurrent_hash_map.h" diff --git a/src/3rd party/tbb/concurrent_lru_cache.h b/src/3rd party/tbb/concurrent_lru_cache.h index c743e41e9d..2757a234be 100644 --- a/src/3rd party/tbb/concurrent_lru_cache.h +++ b/src/3rd party/tbb/concurrent_lru_cache.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,277 +14,4 @@ limitations under the License. */ -#ifndef __TBB_concurrent_lru_cache_H -#define __TBB_concurrent_lru_cache_H - -#define __TBB_concurrent_lru_cache_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#if ! TBB_PREVIEW_CONCURRENT_LRU_CACHE - #error Set TBB_PREVIEW_CONCURRENT_LRU_CACHE to include concurrent_lru_cache.h -#endif - -#include "tbb_stddef.h" - -#include -#include -#include // std::find -#if __TBB_CPP11_RVALUE_REF_PRESENT -#include // std::move -#endif - -#include "atomic.h" -#include "internal/_aggregator_impl.h" - -namespace tbb{ -namespace interface6 { - - -template -class concurrent_lru_cache : internal::no_assign{ -private: - typedef concurrent_lru_cache self_type; - typedef value_functor_type value_function_type; - typedef std::size_t ref_counter_type; - struct map_value_type; - typedef std::map map_storage_type; - typedef std::list lru_list_type; - struct map_value_type { - value_type my_value; - ref_counter_type my_ref_counter; - typename lru_list_type::iterator my_lru_list_iterator; - bool my_is_ready; - - map_value_type (value_type const& a_value, ref_counter_type a_ref_counter, typename lru_list_type::iterator a_lru_list_iterator, bool a_is_ready) - : my_value(a_value), my_ref_counter(a_ref_counter), my_lru_list_iterator (a_lru_list_iterator), my_is_ready(a_is_ready) - {} - }; - - class handle_object; - - struct aggregator_operation; - typedef aggregator_operation aggregated_operation_type; - typedef tbb::internal::aggregating_functor aggregator_function_type; - friend class tbb::internal::aggregating_functor; - typedef tbb::internal::aggregator aggregator_type; - -private: - value_function_type my_value_function; - std::size_t const my_number_of_lru_history_items; - map_storage_type my_map_storage; - lru_list_type my_lru_list; - aggregator_type my_aggregator; - -public: - typedef handle_object handle; - -public: - concurrent_lru_cache(value_function_type f, std::size_t number_of_lru_history_items) - : my_value_function(f),my_number_of_lru_history_items(number_of_lru_history_items) - { - my_aggregator.initialize_handler(aggregator_function_type(this)); - } - - handle_object operator[](key_type k){ - retrieve_aggregator_operation op(k); - my_aggregator.execute(&op); - if (op.is_new_value_needed()){ - op.result().second.my_value = my_value_function(k); - __TBB_store_with_release(op.result().second.my_is_ready, true); - }else{ - tbb::internal::spin_wait_while_eq(op.result().second.my_is_ready,false); - } - return handle_object(*this,op.result()); - } -private: - void signal_end_of_usage(typename map_storage_type::reference value_ref){ - signal_end_of_usage_aggregator_operation op(value_ref); - my_aggregator.execute(&op); - } - -private: -#if !__TBB_CPP11_RVALUE_REF_PRESENT - struct handle_move_t:no_assign{ - concurrent_lru_cache & my_cache_ref; - typename map_storage_type::reference my_map_record_ref; - handle_move_t(concurrent_lru_cache & cache_ref, typename map_storage_type::reference value_ref):my_cache_ref(cache_ref),my_map_record_ref(value_ref) {}; - }; -#endif - class handle_object { - concurrent_lru_cache * my_cache_pointer; - typename map_storage_type::pointer my_map_record_ptr; - public: - handle_object() : my_cache_pointer(), my_map_record_ptr() {} - handle_object(concurrent_lru_cache& cache_ref, typename map_storage_type::reference value_ref) : my_cache_pointer(&cache_ref), my_map_record_ptr(&value_ref) {} - operator bool() const { - return (my_cache_pointer && my_map_record_ptr); - } -#if __TBB_CPP11_RVALUE_REF_PRESENT - // TODO: add check for double moved objects by special dedicated field - handle_object(handle_object&& src) : my_cache_pointer(src.my_cache_pointer), my_map_record_ptr(src.my_map_record_ptr) { - __TBB_ASSERT((src.my_cache_pointer && src.my_map_record_ptr) || (!src.my_cache_pointer && !src.my_map_record_ptr), "invalid state of moving object?"); - src.my_cache_pointer = NULL; - src.my_map_record_ptr = NULL; - } - handle_object& operator=(handle_object&& src) { - __TBB_ASSERT((src.my_cache_pointer && src.my_map_record_ptr) || (!src.my_cache_pointer && !src.my_map_record_ptr), "invalid state of moving object?"); - if (my_cache_pointer) { - my_cache_pointer->signal_end_of_usage(*my_map_record_ptr); - } - my_cache_pointer = src.my_cache_pointer; - my_map_record_ptr = src.my_map_record_ptr; - src.my_cache_pointer = NULL; - src.my_map_record_ptr = NULL; - return *this; - } -#else - handle_object(handle_move_t m) : my_cache_pointer(&m.my_cache_ref), my_map_record_ptr(&m.my_map_record_ref) {} - handle_object& operator=(handle_move_t m) { - if (my_cache_pointer) { - my_cache_pointer->signal_end_of_usage(*my_map_record_ptr); - } - my_cache_pointer = &m.my_cache_ref; - my_map_record_ptr = &m.my_map_record_ref; - return *this; - } - operator handle_move_t(){ - return move(*this); - } -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - value_type& value(){ - __TBB_ASSERT(my_cache_pointer,"get value from already moved object?"); - __TBB_ASSERT(my_map_record_ptr,"get value from an invalid or already moved object?"); - return my_map_record_ptr->second.my_value; - } - ~handle_object(){ - if (my_cache_pointer){ - my_cache_pointer->signal_end_of_usage(*my_map_record_ptr); - } - } - private: -#if __TBB_CPP11_RVALUE_REF_PRESENT - // For source compatibility with C++03 - friend handle_object&& move(handle_object& h){ - return std::move(h); - } -#else - friend handle_move_t move(handle_object& h){ - return handle_object::move(h); - } - // TODO: add check for double moved objects by special dedicated field - static handle_move_t move(handle_object& h){ - __TBB_ASSERT((h.my_cache_pointer && h.my_map_record_ptr) || (!h.my_cache_pointer && !h.my_map_record_ptr), "invalid state of moving object?"); - concurrent_lru_cache * cache_pointer = h.my_cache_pointer; - typename map_storage_type::pointer map_record_ptr = h.my_map_record_ptr; - h.my_cache_pointer = NULL; - h.my_map_record_ptr = NULL; - return handle_move_t(*cache_pointer, *map_record_ptr); - } -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - private: - void operator=(handle_object&); -#if __SUNPRO_CC - // Presumably due to a compiler error, private copy constructor - // breaks expressions like handle h = cache[key]; - public: -#endif - handle_object(handle_object &); - }; -private: - //TODO: looks like aggregator_operation is a perfect match for statically typed variant type - struct aggregator_operation : tbb::internal::aggregated_operation{ - enum e_op_type {op_retive, op_signal_end_of_usage}; - //TODO: try to use pointer to function apply_visitor here - //TODO: try virtual functions and measure the difference - e_op_type my_operation_type; - aggregator_operation(e_op_type operation_type): my_operation_type(operation_type) {} - void cast_and_handle(self_type& container ){ - if (my_operation_type==op_retive){ - static_cast(this)->handle(container); - }else{ - static_cast(this)->handle(container); - } - } - }; - struct retrieve_aggregator_operation : aggregator_operation, private internal::no_assign { - key_type my_key; - typename map_storage_type::pointer my_result_map_record_pointer; - bool my_is_new_value_needed; - retrieve_aggregator_operation(key_type key):aggregator_operation(aggregator_operation::op_retive),my_key(key),my_is_new_value_needed(false){} - void handle(self_type& container ){ - my_result_map_record_pointer = & container.retrieve_serial(my_key,my_is_new_value_needed); - } - typename map_storage_type::reference result(){ return * my_result_map_record_pointer; } - bool is_new_value_needed(){return my_is_new_value_needed;} - }; - struct signal_end_of_usage_aggregator_operation : aggregator_operation, private internal::no_assign { - typename map_storage_type::reference my_map_record_ref; - signal_end_of_usage_aggregator_operation(typename map_storage_type::reference map_record_ref):aggregator_operation(aggregator_operation::op_signal_end_of_usage),my_map_record_ref(map_record_ref){} - void handle(self_type& container ){ - container.signal_end_of_usage_serial(my_map_record_ref); - } - }; - -private: - void handle_operations(aggregator_operation* op_list){ - while(op_list){ - op_list->cast_and_handle(*this); - aggregator_operation* tmp = op_list; - op_list=op_list->next; - tbb::internal::itt_store_word_with_release(tmp->status, uintptr_t(1)); - } - } - -private: - typename map_storage_type::reference retrieve_serial(key_type k, bool& is_new_value_needed){ - typename map_storage_type::iterator it = my_map_storage.find(k); - if (it == my_map_storage.end()){ - it = my_map_storage.insert(it,std::make_pair(k,map_value_type(value_type(),0,my_lru_list.end(),false))); - is_new_value_needed = true; - }else { - typename lru_list_type::iterator list_it = it->second.my_lru_list_iterator; - if (list_it!=my_lru_list.end()) { - __TBB_ASSERT(!it->second.my_ref_counter,"item to be evicted should not have a live references"); - //item is going to be used. Therefore it is not a subject for eviction - //so - remove it from LRU history. - my_lru_list.erase(list_it); - it->second.my_lru_list_iterator= my_lru_list.end(); - } - } - ++(it->second.my_ref_counter); - return *it; - } - - void signal_end_of_usage_serial(typename map_storage_type::reference map_record_ref){ - typename map_storage_type::iterator it = my_map_storage.find(map_record_ref.first); - __TBB_ASSERT(it!=my_map_storage.end(),"cache should not return past-end iterators to outer world"); - __TBB_ASSERT(&(*it) == &map_record_ref,"dangling reference has been returned to outside world? data race ?"); - __TBB_ASSERT( my_lru_list.end()== std::find(my_lru_list.begin(),my_lru_list.end(),it), - "object in use should not be in list of unused objects "); - if (! --(it->second.my_ref_counter)){ - //it was the last reference so put it to the LRU history - if (my_lru_list.size()>=my_number_of_lru_history_items){ - //evict items in order to get a space - size_t number_of_elements_to_evict = 1 + my_lru_list.size() - my_number_of_lru_history_items; - for (size_t i=0; isecond.my_ref_counter,"item to be evicted should not have a live references"); - my_lru_list.pop_back(); - my_map_storage.erase(it_to_evict); - } - } - my_lru_list.push_front(it); - it->second.my_lru_list_iterator = my_lru_list.begin(); - } - } -}; -} // namespace interface6 - -using interface6::concurrent_lru_cache; - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_concurrent_lru_cache_H_include_area - -#endif //__TBB_concurrent_lru_cache_H +#include "../oneapi/tbb/concurrent_lru_cache.h" diff --git a/src/3rd party/tbb/concurrent_map.h b/src/3rd party/tbb/concurrent_map.h index 17b9d07ca6..84f59d7e66 100644 --- a/src/3rd party/tbb/concurrent_map.h +++ b/src/3rd party/tbb/concurrent_map.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2019 Intel Corporation + Copyright (c) 2019-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,376 +14,4 @@ limitations under the License. */ -#ifndef __TBB_concurrent_map_H -#define __TBB_concurrent_map_H - -#define __TBB_concurrent_map_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#if !TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS -#error Set TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS to include concurrent_map.h -#endif - -#include "tbb_config.h" - -// concurrent_map requires C++11 support -#if __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT - -#include "internal/_concurrent_skip_list_impl.h" - -namespace tbb { - -namespace interface10 { - -template -class map_traits { -public: - static constexpr size_t MAX_LEVEL = MAX_LEVELS; - using random_level_generator_type = RandomGenerator; - using key_type = Key; - using mapped_type = Value; - using compare_type = KeyCompare; - using value_type = std::pair; - using reference = value_type&; - using const_reference = const value_type&; - using allocator_type = Allocator; - using mutex_type = tbb::spin_mutex; - using node_type = tbb::internal::node_handle, allocator_type>; - - static const bool allow_multimapping = AllowMultimapping; - - class value_compare { - public: - // TODO: these member types are deprecated in C++17, do we need to let them - using result_type = bool; - using first_argument_type = value_type; - using second_argument_type = value_type; - - bool operator()(const value_type& lhs, const value_type& rhs) const { - return comp(lhs.first, rhs.first); - } - - protected: - value_compare(compare_type c) : comp(c) {} - - friend class map_traits; - - compare_type comp; - }; - - static value_compare value_comp(compare_type comp) { return value_compare(comp); } - - static const key_type& get_key(const_reference val) { - return val.first; - } -}; // class map_traits - -template -class concurrent_multimap; - -template , typename Allocator = tbb_allocator>> -class concurrent_map - : public internal::concurrent_skip_list, 64, Allocator, false>> { - using traits_type = map_traits, 64, Allocator, false>; - using base_type = internal::concurrent_skip_list; -#if __TBB_EXTRA_DEBUG -public: -#endif - using base_type::allow_multimapping; -public: - using key_type = Key; - using mapped_type = Value; - using value_type = typename traits_type::value_type; - using size_type = typename base_type::size_type; - using difference_type = typename base_type::difference_type; - using key_compare = Comp; - using value_compare = typename base_type::value_compare; - using allocator_type = Allocator; - - using reference = typename base_type::reference; - using const_reference = typename base_type::const_reference; - using pointer = typename base_type::pointer; - using const_pointer = typename base_type::pointer; - - using iterator = typename base_type::iterator; - using const_iterator = typename base_type::const_iterator; - using reverse_iterator = typename base_type::reverse_iterator; - using const_reverse_iterator = typename base_type::const_reverse_iterator; - - using node_type = typename base_type::node_type; - - using base_type::end; - using base_type::find; - using base_type::emplace; - using base_type::insert; - - concurrent_map() = default; - - explicit concurrent_map(const key_compare& comp, const allocator_type& alloc = allocator_type()) : base_type(comp, alloc) {} - - explicit concurrent_map(const allocator_type& alloc) : base_type(key_compare(), alloc) {} - - template< class InputIt > - concurrent_map(InputIt first, InputIt last, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) - : base_type(first, last, comp, alloc) {} - - template< class InputIt > - concurrent_map(InputIt first, InputIt last, const allocator_type& alloc) : base_type(first, last, key_compare(), alloc) {} - - /** Copy constructor */ - concurrent_map(const concurrent_map&) = default; - - concurrent_map(const concurrent_map& other, const allocator_type& alloc) : base_type(other, alloc) {} - - concurrent_map(concurrent_map&&) = default; - - concurrent_map(concurrent_map&& other, const allocator_type& alloc) : base_type(std::move(other), alloc) {} - - concurrent_map(std::initializer_list init, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) - : base_type(comp, alloc) { - insert(init); - } - - concurrent_map(std::initializer_list init, const allocator_type& alloc) - : base_type(key_compare(), alloc) { - insert(init); - } - - concurrent_map& operator=(const concurrent_map& other) { - return static_cast(base_type::operator=(other)); - } - - concurrent_map& operator=(concurrent_map&& other) { - return static_cast(base_type::operator=(std::move(other))); - } - - mapped_type& at(const key_type& key) { - iterator it = find(key); - - if (it == end()) { - tbb::internal::throw_exception(tbb::internal::eid_invalid_key); - } - - return it->second; - } - - const mapped_type& at(const key_type& key) const { - const_iterator it = find(key); - - if (it == end()) { - tbb::internal::throw_exception(tbb::internal::eid_invalid_key); - } - - return it->second; - } - - mapped_type& operator[](const key_type& key) { - iterator it = find(key); - - if (it == end()) { - it = emplace(std::piecewise_construct, std::forward_as_tuple(key), std::tuple<>()).first; - } - - return it->second; - } - - mapped_type& operator[](key_type&& key) { - iterator it = find(key); - - if (it == end()) { - it = emplace(std::piecewise_construct, std::forward_as_tuple(std::move(key)), std::tuple<>()).first; - } - - return it->second; - } - - template::value>::type> - std::pair insert(P&& value) { - return emplace(std::forward

(value)); - } - - template::value>::type> - iterator insert(const_iterator hint, P&& value) { - return emplace_hint(hint, std::forward

(value)); - return end(); - } - - template - void merge(concurrent_map& source) { - this->internal_merge(source); - } - - template - void merge(concurrent_map&& source) { - this->internal_merge(std::move(source)); - } - - template - void merge(concurrent_multimap& source) { - this->internal_merge(source); - } - - template - void merge(concurrent_multimap&& source) { - this->internal_merge(std::move(source)); - } -}; // class concurrent_map - -#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT - -namespace internal { - -using namespace tbb::internal; - -template typename Map, typename Key, typename T, typename... Args> -using c_map_t = Map 0) && !is_allocator_v >, - pack_element_t<0, Args...>, std::less >, - std::conditional_t< (sizeof...(Args) > 0) && is_allocator_v >, - pack_element_t, tbb_allocator > > >; -} // namespace internal - -template -concurrent_map(It, It, Args...) --> internal::c_map_t, internal::iterator_mapped_t, Args...>; - -template -concurrent_map(std::initializer_list>, Args...) --> internal::c_map_t; - -#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT - -template , typename Allocator = tbb_allocator>> -class concurrent_multimap - : public internal::concurrent_skip_list, 64, Allocator, true>> { - using traits_type = map_traits, 64, Allocator, true>; - using base_type = internal::concurrent_skip_list; -#if __TBB_EXTRA_DEBUG -public: -#endif - using base_type::allow_multimapping; -public: - using key_type = Key; - using mapped_type = Value; - using value_type = typename traits_type::value_type; - using size_type = typename base_type::size_type; - using difference_type = typename base_type::difference_type; - using key_compare = Comp; - using value_compare = typename base_type::value_compare; - using allocator_type = Allocator; - - using reference = typename base_type::reference; - using const_reference = typename base_type::const_reference; - using pointer = typename base_type::pointer; - using const_pointer = typename base_type::pointer; - - using iterator = typename base_type::iterator; - using const_iterator = typename base_type::const_iterator; - using reverse_iterator = typename base_type::reverse_iterator; - using const_reverse_iterator = typename base_type::const_reverse_iterator; - - using node_type = typename base_type::node_type; - - using base_type::end; - using base_type::find; - using base_type::emplace; - using base_type::insert; - - concurrent_multimap() = default; - - explicit concurrent_multimap(const key_compare& comp, const allocator_type& alloc = allocator_type()) : base_type(comp, alloc) {} - - explicit concurrent_multimap(const allocator_type& alloc) : base_type(key_compare(), alloc) {} - - template< class InputIt > - concurrent_multimap(InputIt first, InputIt last, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) - : base_type(first, last, comp, alloc) {} - - template< class InputIt > - concurrent_multimap(InputIt first, InputIt last, const allocator_type& alloc) : base_type(first, last, key_compare(), alloc) {} - - /** Copy constructor */ - concurrent_multimap(const concurrent_multimap&) = default; - - concurrent_multimap(const concurrent_multimap& other, const allocator_type& alloc) : base_type(other, alloc) {} - - concurrent_multimap(concurrent_multimap&&) = default; - - concurrent_multimap(concurrent_multimap&& other, const allocator_type& alloc) : base_type(std::move(other), alloc) {} - - concurrent_multimap(std::initializer_list init, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) - : base_type(comp, alloc) { - insert(init); - } - - concurrent_multimap(std::initializer_list init, const allocator_type& alloc) - : base_type(key_compare(), alloc) { - insert(init); - } - - concurrent_multimap& operator=(const concurrent_multimap& other) { - return static_cast(base_type::operator=(other)); - } - - concurrent_multimap& operator=(concurrent_multimap&& other) { - return static_cast(base_type::operator=(std::move(other))); - } - - template::value>::type> - std::pair insert(P&& value) { - return emplace(std::forward

(value)); - } - - template::value>::type> - iterator insert(const_iterator hint, P&& value) { - return emplace_hint(hint, std::forward

(value)); - return end(); - } - - template - void merge(concurrent_multimap& source) { - this->internal_merge(source); - } - - template - void merge(concurrent_multimap&& source) { - this->internal_merge(std::move(source)); - } - - template - void merge(concurrent_map& source) { - this->internal_merge(source); - } - - template - void merge(concurrent_map&& source) { - this->internal_merge(std::move(source)); - } - -}; // class concurrent_multimap - -#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT - -template -concurrent_multimap(It, It, Args...) --> internal::c_map_t, internal::iterator_mapped_t, Args...>; - -template -concurrent_multimap(std::initializer_list>, Args...) --> internal::c_map_t; - -#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT - -} // namespace interface10 - -using interface10::concurrent_map; -using interface10::concurrent_multimap; - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_concurrent_map_H_include_area - -#endif // __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT -#endif // __TBB_concurrent_map_H +#include "../oneapi/tbb/concurrent_map.h" diff --git a/src/3rd party/tbb/concurrent_priority_queue.h b/src/3rd party/tbb/concurrent_priority_queue.h index 97afa2f920..3b27130b1e 100644 --- a/src/3rd party/tbb/concurrent_priority_queue.h +++ b/src/3rd party/tbb/concurrent_priority_queue.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,539 +14,4 @@ limitations under the License. */ -#ifndef __TBB_concurrent_priority_queue_H -#define __TBB_concurrent_priority_queue_H - -#define __TBB_concurrent_priority_queue_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "atomic.h" -#include "cache_aligned_allocator.h" -#include "tbb_exception.h" -#include "tbb_stddef.h" -#include "tbb_profiling.h" -#include "internal/_aggregator_impl.h" -#include "internal/_template_helpers.h" -#include "internal/_allocator_traits.h" -#include -#include -#include -#include __TBB_STD_SWAP_HEADER - -#if __TBB_INITIALIZER_LISTS_PRESENT - #include -#endif - -#if __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT - #include -#endif - -namespace tbb { -namespace interface5 { -namespace internal { -#if __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT - template::value> - struct use_element_copy_constructor { - typedef tbb::internal::true_type type; - }; - template - struct use_element_copy_constructor { - typedef tbb::internal::false_type type; - }; -#else - template - struct use_element_copy_constructor { - typedef tbb::internal::true_type type; - }; -#endif -} // namespace internal - -using namespace tbb::internal; - -//! Concurrent priority queue -template , typename A=cache_aligned_allocator > -class concurrent_priority_queue { - public: - //! Element type in the queue. - typedef T value_type; - - //! Reference type - typedef T& reference; - - //! Const reference type - typedef const T& const_reference; - - //! Integral type for representing size of the queue. - typedef size_t size_type; - - //! Difference type for iterator - typedef ptrdiff_t difference_type; - - //! Allocator type - typedef A allocator_type; - - //! Constructs a new concurrent_priority_queue with default capacity - explicit concurrent_priority_queue(const allocator_type& a = allocator_type()) : mark(0), my_size(0), compare(), data(a) - { - my_aggregator.initialize_handler(my_functor_t(this)); - } - - //! Constructs a new concurrent_priority_queue with default capacity - explicit concurrent_priority_queue(const Compare& c, const allocator_type& a = allocator_type()) : mark(0), my_size(0), compare(c), data(a) - { - my_aggregator.initialize_handler(my_functor_t(this)); - } - - //! Constructs a new concurrent_priority_queue with init_sz capacity - explicit concurrent_priority_queue(size_type init_capacity, const allocator_type& a = allocator_type()) : - mark(0), my_size(0), compare(), data(a) - { - data.reserve(init_capacity); - my_aggregator.initialize_handler(my_functor_t(this)); - } - - //! Constructs a new concurrent_priority_queue with init_sz capacity - explicit concurrent_priority_queue(size_type init_capacity, const Compare& c, const allocator_type& a = allocator_type()) : - mark(0), my_size(0), compare(c), data(a) - { - data.reserve(init_capacity); - my_aggregator.initialize_handler(my_functor_t(this)); - } - - //! [begin,end) constructor - template - concurrent_priority_queue(InputIterator begin, InputIterator end, const allocator_type& a = allocator_type()) : - mark(0), compare(), data(begin, end, a) - { - my_aggregator.initialize_handler(my_functor_t(this)); - heapify(); - my_size = data.size(); - } - - //! [begin,end) constructor - template - concurrent_priority_queue(InputIterator begin, InputIterator end, const Compare& c, const allocator_type& a = allocator_type()) : - mark(0), compare(c), data(begin, end, a) - { - my_aggregator.initialize_handler(my_functor_t(this)); - heapify(); - my_size = data.size(); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from std::initializer_list - concurrent_priority_queue(std::initializer_list init_list, const allocator_type &a = allocator_type()) : - mark(0), compare(), data(init_list.begin(), init_list.end(), a) - { - my_aggregator.initialize_handler(my_functor_t(this)); - heapify(); - my_size = data.size(); - } - - //! Constructor from std::initializer_list - concurrent_priority_queue(std::initializer_list init_list, const Compare& c, const allocator_type &a = allocator_type()) : - mark(0), compare(c), data(init_list.begin(), init_list.end(), a) - { - my_aggregator.initialize_handler(my_functor_t(this)); - heapify(); - my_size = data.size(); - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - - //! Copy constructor - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - concurrent_priority_queue(const concurrent_priority_queue& src) : mark(src.mark), - my_size(src.my_size), data(src.data.begin(), src.data.end(), src.data.get_allocator()) - { - my_aggregator.initialize_handler(my_functor_t(this)); - heapify(); - } - - //! Copy constructor with specific allocator - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - concurrent_priority_queue(const concurrent_priority_queue& src, const allocator_type& a) : mark(src.mark), - my_size(src.my_size), data(src.data.begin(), src.data.end(), a) - { - my_aggregator.initialize_handler(my_functor_t(this)); - heapify(); - } - - //! Assignment operator - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - concurrent_priority_queue& operator=(const concurrent_priority_queue& src) { - if (this != &src) { - vector_t(src.data.begin(), src.data.end(), src.data.get_allocator()).swap(data); - mark = src.mark; - my_size = src.my_size; - } - return *this; - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructor - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - concurrent_priority_queue(concurrent_priority_queue&& src) : mark(src.mark), - my_size(src.my_size), data(std::move(src.data)) - { - my_aggregator.initialize_handler(my_functor_t(this)); - } - - //! Move constructor with specific allocator - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - concurrent_priority_queue(concurrent_priority_queue&& src, const allocator_type& a) : mark(src.mark), - my_size(src.my_size), -#if __TBB_ALLOCATOR_TRAITS_PRESENT - data(std::move(src.data), a) -#else - // Some early version of C++11 STL vector does not have a constructor of vector(vector&& , allocator). - // It seems that the reason is absence of support of allocator_traits (stateful allocators). - data(a) -#endif //__TBB_ALLOCATOR_TRAITS_PRESENT - { - my_aggregator.initialize_handler(my_functor_t(this)); -#if !__TBB_ALLOCATOR_TRAITS_PRESENT - if (a != src.data.get_allocator()){ - data.reserve(src.data.size()); - data.assign(std::make_move_iterator(src.data.begin()), std::make_move_iterator(src.data.end())); - }else{ - data = std::move(src.data); - } -#endif //!__TBB_ALLOCATOR_TRAITS_PRESENT - } - - //! Move assignment operator - /** This operation is unsafe if there are pending concurrent operations on the src queue. */ - concurrent_priority_queue& operator=( concurrent_priority_queue&& src) { - if (this != &src) { - mark = src.mark; - my_size = src.my_size; -#if !__TBB_ALLOCATOR_TRAITS_PRESENT - if (data.get_allocator() != src.data.get_allocator()){ - vector_t(std::make_move_iterator(src.data.begin()), std::make_move_iterator(src.data.end()), data.get_allocator()).swap(data); - }else -#endif //!__TBB_ALLOCATOR_TRAITS_PRESENT - { - data = std::move(src.data); - } - } - return *this; - } -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - - //! Assign the queue from [begin,end) range, not thread-safe - template - void assign(InputIterator begin, InputIterator end) { - vector_t(begin, end, data.get_allocator()).swap(data); - mark = 0; - my_size = data.size(); - heapify(); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Assign the queue from std::initializer_list, not thread-safe - void assign(std::initializer_list il) { this->assign(il.begin(), il.end()); } - - //! Assign from std::initializer_list, not thread-safe - concurrent_priority_queue& operator=(std::initializer_list il) { - this->assign(il.begin(), il.end()); - return *this; - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - - //! Returns true if empty, false otherwise - /** Returned value may not reflect results of pending operations. - This operation reads shared data and will trigger a race condition. */ - bool empty() const { return size()==0; } - - //! Returns the current number of elements contained in the queue - /** Returned value may not reflect results of pending operations. - This operation reads shared data and will trigger a race condition. */ - size_type size() const { return __TBB_load_with_acquire(my_size); } - - //! Pushes elem onto the queue, increasing capacity of queue if necessary - /** This operation can be safely used concurrently with other push, try_pop or emplace operations. */ - void push(const_reference elem) { -#if __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT - __TBB_STATIC_ASSERT( std::is_copy_constructible::value, "The type is not copy constructible. Copying push operation is impossible." ); -#endif - cpq_operation op_data(elem, PUSH_OP); - my_aggregator.execute(&op_data); - if (op_data.status == FAILED) // exception thrown - throw_exception(eid_bad_alloc); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Pushes elem onto the queue, increasing capacity of queue if necessary - /** This operation can be safely used concurrently with other push, try_pop or emplace operations. */ - void push(value_type &&elem) { - cpq_operation op_data(elem, PUSH_RVALUE_OP); - my_aggregator.execute(&op_data); - if (op_data.status == FAILED) // exception thrown - throw_exception(eid_bad_alloc); - } - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - //! Constructs a new element using args as the arguments for its construction and pushes it onto the queue */ - /** This operation can be safely used concurrently with other push, try_pop or emplace operations. */ - template - void emplace(Args&&... args) { - push(value_type(std::forward(args)...)); - } -#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Gets a reference to and removes highest priority element - /** If a highest priority element was found, sets elem and returns true, - otherwise returns false. - This operation can be safely used concurrently with other push, try_pop or emplace operations. */ - bool try_pop(reference elem) { - cpq_operation op_data(POP_OP); - op_data.elem = &elem; - my_aggregator.execute(&op_data); - return op_data.status==SUCCEEDED; - } - - //! Clear the queue; not thread-safe - /** This operation is unsafe if there are pending concurrent operations on the queue. - Resets size, effectively emptying queue; does not free space. - May not clear elements added in pending operations. */ - void clear() { - data.clear(); - mark = 0; - my_size = 0; - } - - //! Swap this queue with another; not thread-safe - /** This operation is unsafe if there are pending concurrent operations on the queue. */ - void swap(concurrent_priority_queue& q) { - using std::swap; - data.swap(q.data); - swap(mark, q.mark); - swap(my_size, q.my_size); - } - - //! Return allocator object - allocator_type get_allocator() const { return data.get_allocator(); } - - private: - enum operation_type {INVALID_OP, PUSH_OP, POP_OP, PUSH_RVALUE_OP}; - enum operation_status { WAIT=0, SUCCEEDED, FAILED }; - - class cpq_operation : public aggregated_operation { - public: - operation_type type; - union { - value_type *elem; - size_type sz; - }; - cpq_operation(const_reference e, operation_type t) : - type(t), elem(const_cast(&e)) {} - cpq_operation(operation_type t) : type(t) {} - }; - - class my_functor_t { - concurrent_priority_queue *cpq; - public: - my_functor_t() {} - my_functor_t(concurrent_priority_queue *cpq_) : cpq(cpq_) {} - void operator()(cpq_operation* op_list) { - cpq->handle_operations(op_list); - } - }; - - typedef tbb::internal::aggregator< my_functor_t, cpq_operation > aggregator_t; - aggregator_t my_aggregator; - //! Padding added to avoid false sharing - char padding1[NFS_MaxLineSize - sizeof(aggregator_t)]; - //! The point at which unsorted elements begin - size_type mark; - __TBB_atomic size_type my_size; - Compare compare; - //! Padding added to avoid false sharing - char padding2[NFS_MaxLineSize - (2*sizeof(size_type)) - sizeof(Compare)]; - //! Storage for the heap of elements in queue, plus unheapified elements - /** data has the following structure: - - binary unheapified - heap elements - ____|_______|____ - | | | - v v v - [_|...|_|_|...|_| |...| ] - 0 ^ ^ ^ - | | |__capacity - | |__my_size - |__mark - - Thus, data stores the binary heap starting at position 0 through - mark-1 (it may be empty). Then there are 0 or more elements - that have not yet been inserted into the heap, in positions - mark through my_size-1. */ - typedef std::vector vector_t; - vector_t data; - - void handle_operations(cpq_operation *op_list) { - cpq_operation *tmp, *pop_list=NULL; - - __TBB_ASSERT(mark == data.size(), NULL); - - // First pass processes all constant (amortized; reallocation may happen) time pushes and pops. - while (op_list) { - // ITT note: &(op_list->status) tag is used to cover accesses to op_list - // node. This thread is going to handle the operation, and so will acquire it - // and perform the associated operation w/o triggering a race condition; the - // thread that created the operation is waiting on the status field, so when - // this thread is done with the operation, it will perform a - // store_with_release to give control back to the waiting thread in - // aggregator::insert_operation. - call_itt_notify(acquired, &(op_list->status)); - __TBB_ASSERT(op_list->type != INVALID_OP, NULL); - tmp = op_list; - op_list = itt_hide_load_word(op_list->next); - if (tmp->type == POP_OP) { - if (mark < data.size() && - compare(data[0], data[data.size()-1])) { - // there are newly pushed elems and the last one - // is higher than top - *(tmp->elem) = tbb::internal::move(data[data.size()-1]); - __TBB_store_with_release(my_size, my_size-1); - itt_store_word_with_release(tmp->status, uintptr_t(SUCCEEDED)); - data.pop_back(); - __TBB_ASSERT(mark<=data.size(), NULL); - } - else { // no convenient item to pop; postpone - itt_hide_store_word(tmp->next, pop_list); - pop_list = tmp; - } - } else { // PUSH_OP or PUSH_RVALUE_OP - __TBB_ASSERT(tmp->type == PUSH_OP || tmp->type == PUSH_RVALUE_OP, "Unknown operation" ); - __TBB_TRY{ - if (tmp->type == PUSH_OP) { - push_back_helper(*(tmp->elem), typename internal::use_element_copy_constructor::type()); - } else { - data.push_back(tbb::internal::move(*(tmp->elem))); - } - __TBB_store_with_release(my_size, my_size + 1); - itt_store_word_with_release(tmp->status, uintptr_t(SUCCEEDED)); - } __TBB_CATCH(...) { - itt_store_word_with_release(tmp->status, uintptr_t(FAILED)); - } - } - } - - // second pass processes pop operations - while (pop_list) { - tmp = pop_list; - pop_list = itt_hide_load_word(pop_list->next); - __TBB_ASSERT(tmp->type == POP_OP, NULL); - if (data.empty()) { - itt_store_word_with_release(tmp->status, uintptr_t(FAILED)); - } - else { - __TBB_ASSERT(mark<=data.size(), NULL); - if (mark < data.size() && - compare(data[0], data[data.size()-1])) { - // there are newly pushed elems and the last one is - // higher than top - *(tmp->elem) = tbb::internal::move(data[data.size()-1]); - __TBB_store_with_release(my_size, my_size-1); - itt_store_word_with_release(tmp->status, uintptr_t(SUCCEEDED)); - data.pop_back(); - } - else { // extract top and push last element down heap - *(tmp->elem) = tbb::internal::move(data[0]); - __TBB_store_with_release(my_size, my_size-1); - itt_store_word_with_release(tmp->status, uintptr_t(SUCCEEDED)); - reheap(); - } - } - } - - // heapify any leftover pushed elements before doing the next - // batch of operations - if (mark0) mark = 1; - for (; mark>1; - if (!compare(data[parent], to_place)) break; - data[cur_pos] = tbb::internal::move(data[parent]); - cur_pos = parent; - } while( cur_pos ); - data[cur_pos] = tbb::internal::move(to_place); - } - } - - //! Re-heapify after an extraction - /** Re-heapify by pushing last element down the heap from the root. */ - void reheap() { - size_type cur_pos=0, child=1; - - while (child < mark) { - size_type target = child; - if (child+1 < mark && compare(data[child], data[child+1])) - ++target; - // target now has the higher priority child - if (compare(data[target], data[data.size()-1])) break; - data[cur_pos] = tbb::internal::move(data[target]); - cur_pos = target; - child = (cur_pos<<1)+1; - } - if (cur_pos != data.size()-1) - data[cur_pos] = tbb::internal::move(data[data.size()-1]); - data.pop_back(); - if (mark > data.size()) mark = data.size(); - } - - void push_back_helper(const T& t, tbb::internal::true_type) { - data.push_back(t); - } - - void push_back_helper(const T&, tbb::internal::false_type) { - __TBB_ASSERT( false, "The type is not copy constructible. Copying push operation is impossible." ); - } -}; - -#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT -namespace internal { - -template -using priority_queue_t = concurrent_priority_queue< - T, - std::conditional_t< (sizeof...(Args)>0) && !is_allocator_v< pack_element_t<0, Args...> >, - pack_element_t<0, Args...>, std::less >, - std::conditional_t< (sizeof...(Args)>0) && is_allocator_v< pack_element_t >, - pack_element_t, cache_aligned_allocator > ->; -} - -// Deduction guide for the constructor from two iterators -template::value_type, - typename... Args -> concurrent_priority_queue(InputIterator, InputIterator, Args...) --> internal::priority_queue_t; - -template -concurrent_priority_queue(std::initializer_list init_list, CompareOrAllocalor) --> internal::priority_queue_t; - -#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ -} // namespace interface5 - -using interface5::concurrent_priority_queue; - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_concurrent_priority_queue_H_include_area - -#endif /* __TBB_concurrent_priority_queue_H */ +#include "../oneapi/tbb/concurrent_priority_queue.h" diff --git a/src/3rd party/tbb/concurrent_queue.h b/src/3rd party/tbb/concurrent_queue.h index 1e6732551d..d81a58b887 100644 --- a/src/3rd party/tbb/concurrent_queue.h +++ b/src/3rd party/tbb/concurrent_queue.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,466 +14,4 @@ limitations under the License. */ -#ifndef __TBB_concurrent_queue_H -#define __TBB_concurrent_queue_H - -#define __TBB_concurrent_queue_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "internal/_concurrent_queue_impl.h" -#include "internal/_allocator_traits.h" - -namespace tbb { - -namespace strict_ppl { - -//! A high-performance thread-safe non-blocking concurrent queue. -/** Multiple threads may each push and pop concurrently. - Assignment construction is not allowed. - @ingroup containers */ -template > -class concurrent_queue: public internal::concurrent_queue_base_v3 { - template friend class internal::concurrent_queue_iterator; - - //! Allocator type - typedef typename tbb::internal::allocator_rebind::type page_allocator_type; - page_allocator_type my_allocator; - - //! Allocates a block of size n (bytes) - virtual void *allocate_block( size_t n ) __TBB_override { - void *b = reinterpret_cast(my_allocator.allocate( n )); - if( !b ) - internal::throw_exception(internal::eid_bad_alloc); - return b; - } - - //! Deallocates block created by allocate_block. - virtual void deallocate_block( void *b, size_t n ) __TBB_override { - my_allocator.deallocate( reinterpret_cast(b), n ); - } - - static void copy_construct_item(T* location, const void* src){ - new (location) T(*static_cast(src)); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - static void move_construct_item(T* location, const void* src) { - new (location) T( std::move(*static_cast(const_cast(src))) ); - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ -public: - //! Element type in the queue. - typedef T value_type; - - //! Reference type - typedef T& reference; - - //! Const reference type - typedef const T& const_reference; - - //! Integral type for representing size of the queue. - typedef size_t size_type; - - //! Difference type for iterator - typedef ptrdiff_t difference_type; - - //! Allocator type - typedef A allocator_type; - - //! Construct empty queue - explicit concurrent_queue(const allocator_type& a = allocator_type()) : - my_allocator( a ) - { - } - - //! [begin,end) constructor - template - concurrent_queue( InputIterator begin, InputIterator end, const allocator_type& a = allocator_type()) : - my_allocator( a ) - { - for( ; begin != end; ++begin ) - this->push(*begin); - } - - //! Copy constructor - concurrent_queue( const concurrent_queue& src, const allocator_type& a = allocator_type()) : - internal::concurrent_queue_base_v3(), my_allocator( a ) - { - this->assign( src, copy_construct_item ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructors - concurrent_queue( concurrent_queue&& src ) : - internal::concurrent_queue_base_v3(), my_allocator( std::move(src.my_allocator) ) - { - this->internal_swap( src ); - } - - concurrent_queue( concurrent_queue&& src, const allocator_type& a ) : - internal::concurrent_queue_base_v3(), my_allocator( a ) - { - // checking that memory allocated by one instance of allocator can be deallocated - // with another - if( my_allocator == src.my_allocator) { - this->internal_swap( src ); - } else { - // allocators are different => performing per-element move - this->assign( src, move_construct_item ); - src.clear(); - } - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Destroy queue - ~concurrent_queue(); - - //! Enqueue an item at tail of queue. - void push( const T& source ) { - this->internal_push( &source, copy_construct_item ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - void push( T&& source ) { - this->internal_push( &source, move_construct_item ); - } - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - void emplace( Arguments&&... args ) { - push( T(std::forward( args )...) ); - } -#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Attempt to dequeue an item from head of queue. - /** Does not wait for item to become available. - Returns true if successful; false otherwise. */ - bool try_pop( T& result ) { - return this->internal_try_pop( &result ); - } - - //! Return the number of items in the queue; thread unsafe - size_type unsafe_size() const {return this->internal_size();} - - //! Equivalent to size()==0. - bool empty() const {return this->internal_empty();} - - //! Clear the queue. not thread-safe. - void clear() ; - - //! Return allocator object - allocator_type get_allocator() const { return this->my_allocator; } - - typedef internal::concurrent_queue_iterator iterator; - typedef internal::concurrent_queue_iterator const_iterator; - - //------------------------------------------------------------------------ - // The iterators are intended only for debugging. They are slow and not thread safe. - //------------------------------------------------------------------------ - iterator unsafe_begin() {return iterator(*this);} - iterator unsafe_end() {return iterator();} - const_iterator unsafe_begin() const {return const_iterator(*this);} - const_iterator unsafe_end() const {return const_iterator();} -} ; - -#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT -// Deduction guide for the constructor from two iterators -template::value_type, - typename A = cache_aligned_allocator -> concurrent_queue(InputIterator, InputIterator, const A& = A()) --> concurrent_queue; -#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ - -template -concurrent_queue::~concurrent_queue() { - clear(); - this->internal_finish_clear(); -} - -template -void concurrent_queue::clear() { - T value; - while( !empty() ) try_pop(value); -} - -} // namespace strict_ppl - -//! A high-performance thread-safe blocking concurrent bounded queue. -/** This is the pre-PPL TBB concurrent queue which supports boundedness and blocking semantics. - Note that method names agree with the PPL-style concurrent queue. - Multiple threads may each push and pop concurrently. - Assignment construction is not allowed. - @ingroup containers */ -template > -class concurrent_bounded_queue: public internal::concurrent_queue_base_v8 { - template friend class internal::concurrent_queue_iterator; - typedef typename tbb::internal::allocator_rebind::type page_allocator_type; - - //! Allocator type - page_allocator_type my_allocator; - - typedef typename concurrent_queue_base_v3::padded_page padded_page; - typedef typename concurrent_queue_base_v3::copy_specifics copy_specifics; - - //! Class used to ensure exception-safety of method "pop" - class destroyer: internal::no_copy { - T& my_value; - public: - destroyer( T& value ) : my_value(value) {} - ~destroyer() {my_value.~T();} - }; - - T& get_ref( page& p, size_t index ) { - __TBB_ASSERT( index(static_cast(&p))->last)[index]; - } - - virtual void copy_item( page& dst, size_t index, const void* src ) __TBB_override { - new( &get_ref(dst,index) ) T(*static_cast(src)); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - virtual void move_item( page& dst, size_t index, const void* src ) __TBB_override { - new( &get_ref(dst,index) ) T( std::move(*static_cast(const_cast(src))) ); - } -#else - virtual void move_item( page&, size_t, const void* ) __TBB_override { - __TBB_ASSERT( false, "Unreachable code" ); - } -#endif - - virtual void copy_page_item( page& dst, size_t dindex, const page& src, size_t sindex ) __TBB_override { - new( &get_ref(dst,dindex) ) T( get_ref( const_cast(src), sindex ) ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - virtual void move_page_item( page& dst, size_t dindex, const page& src, size_t sindex ) __TBB_override { - new( &get_ref(dst,dindex) ) T( std::move(get_ref( const_cast(src), sindex )) ); - } -#else - virtual void move_page_item( page&, size_t, const page&, size_t ) __TBB_override { - __TBB_ASSERT( false, "Unreachable code" ); - } -#endif - - virtual void assign_and_destroy_item( void* dst, page& src, size_t index ) __TBB_override { - T& from = get_ref(src,index); - destroyer d(from); - *static_cast(dst) = tbb::internal::move( from ); - } - - virtual page *allocate_page() __TBB_override { - size_t n = sizeof(padded_page) + (items_per_page-1)*sizeof(T); - page *p = reinterpret_cast(my_allocator.allocate( n )); - if( !p ) - internal::throw_exception(internal::eid_bad_alloc); - return p; - } - - virtual void deallocate_page( page *p ) __TBB_override { - size_t n = sizeof(padded_page) + (items_per_page-1)*sizeof(T); - my_allocator.deallocate( reinterpret_cast(p), n ); - } - -public: - //! Element type in the queue. - typedef T value_type; - - //! Allocator type - typedef A allocator_type; - - //! Reference type - typedef T& reference; - - //! Const reference type - typedef const T& const_reference; - - //! Integral type for representing size of the queue. - /** Note that the size_type is a signed integral type. - This is because the size can be negative if there are pending pops without corresponding pushes. */ - typedef std::ptrdiff_t size_type; - - //! Difference type for iterator - typedef std::ptrdiff_t difference_type; - - //! Construct empty queue - explicit concurrent_bounded_queue(const allocator_type& a = allocator_type()) : - concurrent_queue_base_v8( sizeof(T) ), my_allocator( a ) - { - } - - //! Copy constructor - concurrent_bounded_queue( const concurrent_bounded_queue& src, const allocator_type& a = allocator_type()) - : concurrent_queue_base_v8( sizeof(T) ), my_allocator( a ) - { - assign( src ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructors - concurrent_bounded_queue( concurrent_bounded_queue&& src ) - : concurrent_queue_base_v8( sizeof(T) ), my_allocator( std::move(src.my_allocator) ) - { - internal_swap( src ); - } - - concurrent_bounded_queue( concurrent_bounded_queue&& src, const allocator_type& a ) - : concurrent_queue_base_v8( sizeof(T) ), my_allocator( a ) - { - // checking that memory allocated by one instance of allocator can be deallocated - // with another - if( my_allocator == src.my_allocator) { - this->internal_swap( src ); - } else { - // allocators are different => performing per-element move - this->move_content( src ); - src.clear(); - } - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! [begin,end) constructor - template - concurrent_bounded_queue( InputIterator begin, InputIterator end, - const allocator_type& a = allocator_type()) - : concurrent_queue_base_v8( sizeof(T) ), my_allocator( a ) - { - for( ; begin != end; ++begin ) - internal_push_if_not_full(&*begin); - } - - //! Destroy queue - ~concurrent_bounded_queue(); - - //! Enqueue an item at tail of queue. - void push( const T& source ) { - internal_push( &source ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move an item at tail of queue. - void push( T&& source ) { - internal_push_move( &source ); - } - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - void emplace( Arguments&&... args ) { - push( T(std::forward( args )...) ); - } -#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Dequeue item from head of queue. - /** Block until an item becomes available, and then dequeue it. */ - void pop( T& destination ) { - internal_pop( &destination ); - } - -#if TBB_USE_EXCEPTIONS - //! Abort all pending queue operations - void abort() { - internal_abort(); - } -#endif - - //! Enqueue an item at tail of queue if queue is not already full. - /** Does not wait for queue to become not full. - Returns true if item is pushed; false if queue was already full. */ - bool try_push( const T& source ) { - return internal_push_if_not_full( &source ); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move an item at tail of queue if queue is not already full. - /** Does not wait for queue to become not full. - Returns true if item is pushed; false if queue was already full. */ - bool try_push( T&& source ) { - return internal_push_move_if_not_full( &source ); - } -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - bool try_emplace( Arguments&&... args ) { - return try_push( T(std::forward( args )...) ); - } -#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Attempt to dequeue an item from head of queue. - /** Does not wait for item to become available. - Returns true if successful; false otherwise. */ - bool try_pop( T& destination ) { - return internal_pop_if_present( &destination ); - } - - //! Return number of pushes minus number of pops. - /** Note that the result can be negative if there are pops waiting for the - corresponding pushes. The result can also exceed capacity() if there - are push operations in flight. */ - size_type size() const {return internal_size();} - - //! Equivalent to size()<=0. - bool empty() const {return internal_empty();} - - //! Maximum number of allowed elements - size_type capacity() const { - return my_capacity; - } - - //! Set the capacity - /** Setting the capacity to 0 causes subsequent try_push operations to always fail, - and subsequent push operations to block forever. */ - void set_capacity( size_type new_capacity ) { - internal_set_capacity( new_capacity, sizeof(T) ); - } - - //! return allocator object - allocator_type get_allocator() const { return this->my_allocator; } - - //! clear the queue. not thread-safe. - void clear() ; - - typedef internal::concurrent_queue_iterator iterator; - typedef internal::concurrent_queue_iterator const_iterator; - - //------------------------------------------------------------------------ - // The iterators are intended only for debugging. They are slow and not thread safe. - //------------------------------------------------------------------------ - iterator unsafe_begin() {return iterator(*this);} - iterator unsafe_end() {return iterator();} - const_iterator unsafe_begin() const {return const_iterator(*this);} - const_iterator unsafe_end() const {return const_iterator();} - -}; - -#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT -// guide for concurrent_bounded_queue(InputIterator, InputIterator, ...) -template::value_type, - typename A = cache_aligned_allocator -> concurrent_bounded_queue(InputIterator, InputIterator, const A& = A()) --> concurrent_bounded_queue; -#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ - -template -concurrent_bounded_queue::~concurrent_bounded_queue() { - clear(); - internal_finish_clear(); -} - -template -void concurrent_bounded_queue::clear() { - T value; - while( try_pop(value) ) /*noop*/; -} - -using strict_ppl::concurrent_queue; - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_concurrent_queue_H_include_area - -#endif /* __TBB_concurrent_queue_H */ +#include "../oneapi/tbb/concurrent_queue.h" diff --git a/src/3rd party/tbb/concurrent_set.h b/src/3rd party/tbb/concurrent_set.h index 69c9d2e494..cf4652f597 100644 --- a/src/3rd party/tbb/concurrent_set.h +++ b/src/3rd party/tbb/concurrent_set.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2019 Intel Corporation + Copyright (c) 2019-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,291 +14,4 @@ limitations under the License. */ -#ifndef __TBB_concurrent_set_H -#define __TBB_concurrent_set_H - -#define __TBB_concurrent_set_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#if !TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS -#error Set TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS to include concurrent_set.h -#endif - -#include "tbb/tbb_config.h" - -// concurrent_set requires C++11 support -#if __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT - -#include "internal/_concurrent_skip_list_impl.h" - -namespace tbb { -namespace interface10 { - -// TODO: test this class -template -class set_traits { -public: - static constexpr size_t MAX_LEVEL = MAX_LEVELS; - using random_level_generator_type = RandomGenerator; - using key_type = Key; - using value_type = key_type; - using compare_type = KeyCompare; - using value_compare = compare_type; - using reference = value_type & ; - using const_reference = const value_type&; - using allocator_type = Allocator; - using mutex_type = tbb::spin_mutex; - using node_type = tbb::internal::node_handle, allocator_type>; - - static const bool allow_multimapping = AllowMultimapping; - - static const key_type& get_key(const_reference val) { - return val; - } - - static value_compare value_comp(compare_type comp) { return comp; } -}; - -template -class concurrent_multiset; - -template , typename Allocator = tbb_allocator> -class concurrent_set - : public internal::concurrent_skip_list, 64, Allocator, false>> { - using traits_type = set_traits, 64, Allocator, false>; - using base_type = internal::concurrent_skip_list; -#if __TBB_EXTRA_DEBUG -public: -#endif - using base_type::allow_multimapping; -public: - using key_type = Key; - using value_type = typename traits_type::value_type; - using size_type = typename base_type::size_type; - using difference_type = typename base_type::difference_type; - using key_compare = Comp; - using value_compare = typename base_type::value_compare; - using allocator_type = Allocator; - - using reference = typename base_type::reference; - using const_reference = typename base_type::const_reference; - using pointer = typename base_type::pointer; - using const_pointer = typename base_type::pointer; - - using iterator = typename base_type::iterator; - using const_iterator = typename base_type::const_iterator; - using reverse_iterator = typename base_type::reverse_iterator; - using const_reverse_iterator = typename base_type::const_reverse_iterator; - - using node_type = typename base_type::node_type; - - using base_type::insert; - - concurrent_set() = default; - - explicit concurrent_set(const key_compare& comp, const allocator_type& alloc = allocator_type()) : base_type(comp, alloc) {} - - explicit concurrent_set(const allocator_type& alloc) : base_type(key_compare(), alloc) {} - - template< class InputIt > - concurrent_set(InputIt first, InputIt last, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) - : base_type(first, last, comp, alloc) {} - - template< class InputIt > - concurrent_set(InputIt first, InputIt last, const allocator_type& alloc) : base_type(first, last, key_compare(), alloc) {} - - /** Copy constructor */ - concurrent_set(const concurrent_set&) = default; - - concurrent_set(const concurrent_set& other, const allocator_type& alloc) : base_type(other, alloc) {} - - concurrent_set(concurrent_set&&) = default; - - concurrent_set(concurrent_set&& other, const allocator_type& alloc) : base_type(std::move(other), alloc) {} - - concurrent_set(std::initializer_list init, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) - : base_type(comp, alloc) { - insert(init); - } - - concurrent_set(std::initializer_list init, const allocator_type& alloc) - : base_type(key_compare(), alloc) { - insert(init); - } - - concurrent_set& operator=(const concurrent_set& other) { - return static_cast(base_type::operator=(other)); - } - - concurrent_set& operator=(concurrent_set&& other) { - return static_cast(base_type::operator=(std::move(other))); - } - - template - void merge(concurrent_set& source) { - this->internal_merge(source); - } - - template - void merge(concurrent_set&& source) { - this->internal_merge(std::move(source)); - } - - template - void merge(concurrent_multiset& source) { - this->internal_merge(source); - } - - template - void merge(concurrent_multiset&& source) { - this->internal_merge(std::move(source)); - } -}; // class concurrent_set - -#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT - -namespace internal { - -using namespace tbb::internal; - -template typename Set, typename Key, typename... Args> -using c_set_t = Set 0) && !is_allocator_v >, - pack_element_t<0, Args...>, std::less >, - std::conditional_t< (sizeof...(Args) > 0) && is_allocator_v >, - pack_element_t, tbb_allocator > >; -} // namespace internal - -template -concurrent_set(It, It, Args...) --> internal::c_set_t, Args...>; - -template -concurrent_set(std::initializer_list, Args...) --> internal::c_set_t; - -#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT - -template , typename Allocator = tbb_allocator> -class concurrent_multiset - : public internal::concurrent_skip_list, 64, Allocator, true>> { - using traits_type = set_traits, 64, Allocator, true>; - using base_type = internal::concurrent_skip_list; -#if __TBB_EXTRA_DEBUG -public: -#endif - using base_type::allow_multimapping; -public: - using key_type = Key; - using value_type = typename traits_type::value_type; - using size_type = typename base_type::size_type; - using difference_type = typename base_type::difference_type; - using key_compare = Comp; - using value_compare = typename base_type::value_compare; - using allocator_type = Allocator; - - using reference = typename base_type::reference; - using const_reference = typename base_type::const_reference; - using pointer = typename base_type::pointer; - using const_pointer = typename base_type::pointer; - - using iterator = typename base_type::iterator; - using const_iterator = typename base_type::const_iterator; - using reverse_iterator = typename base_type::reverse_iterator; - using const_reverse_iterator = typename base_type::const_reverse_iterator; - - using node_type = typename base_type::node_type; - - using base_type::insert; - - concurrent_multiset() = default; - - explicit concurrent_multiset(const key_compare& comp, const allocator_type& alloc = allocator_type()) : base_type(comp, alloc) {} - - explicit concurrent_multiset(const allocator_type& alloc) : base_type(key_compare(), alloc) {} - - template< class InputIt > - concurrent_multiset(InputIt first, InputIt last, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) - : base_type(comp, alloc) { - insert(first, last); - } - - template< class InputIt > - concurrent_multiset(InputIt first, InputIt last, const allocator_type& alloc) : base_type(key_compare(), alloc) { - insert(first, last); - } - - /** Copy constructor */ - concurrent_multiset(const concurrent_multiset&) = default; - - concurrent_multiset(const concurrent_multiset& other, const allocator_type& alloc) : base_type(other, alloc) {} - - concurrent_multiset(concurrent_multiset&&) = default; - - concurrent_multiset(concurrent_multiset&& other, const allocator_type& alloc) : base_type(std::move(other), alloc) {} - - concurrent_multiset(std::initializer_list init, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) - : base_type(comp, alloc) { - insert(init); - } - - concurrent_multiset(std::initializer_list init, const allocator_type& alloc) - : base_type(key_compare(), alloc) { - insert(init); - } - - concurrent_multiset& operator=(const concurrent_multiset& other) { - return static_cast(base_type::operator=(other)); - } - - concurrent_multiset& operator=(concurrent_multiset&& other) { - return static_cast(base_type::operator=(std::move(other))); - } - - template - void merge(concurrent_set& source) { - this->internal_merge(source); - } - - template - void merge(concurrent_set&& source) { - this->internal_merge(std::move(source)); - } - - template - void merge(concurrent_multiset& source) { - this->internal_merge(source); - } - - template - void merge(concurrent_multiset&& source) { - this->internal_merge(std::move(source)); - } -}; // class concurrent_multiset - -#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT - - -template -concurrent_multiset(It, It, Args...) --> internal::c_set_t, Args...>; - -template -concurrent_multiset(std::initializer_list, Args...) --> internal::c_set_t; - -#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT - -} // namespace interface10 - -using interface10::concurrent_set; -using interface10::concurrent_multiset; - -} // namespace tbb - -#endif // __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_concurrent_set_H_include_area - -#endif // __TBB_concurrent_set_H +#include "../oneapi/tbb/concurrent_set.h" diff --git a/src/3rd party/tbb/concurrent_unordered_map.h b/src/3rd party/tbb/concurrent_unordered_map.h index 094162813e..9475c06cf3 100644 --- a/src/3rd party/tbb/concurrent_unordered_map.h +++ b/src/3rd party/tbb/concurrent_unordered_map.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,479 +14,4 @@ limitations under the License. */ -/* Container implementations in this header are based on PPL implementations - provided by Microsoft. */ - -#ifndef __TBB_concurrent_unordered_map_H -#define __TBB_concurrent_unordered_map_H - -#define __TBB_concurrent_unordered_map_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "internal/_concurrent_unordered_impl.h" - -namespace tbb -{ - -namespace interface5 { - -// Template class for hash map traits -template -class concurrent_unordered_map_traits -{ -protected: - typedef std::pair value_type; - typedef Key key_type; - typedef Hash_compare hash_compare; - typedef typename tbb::internal::allocator_rebind::type allocator_type; -#if __TBB_UNORDERED_NODE_HANDLE_PRESENT - typedef tbb::internal::node_handle::node, - allocator_type> node_type; -#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT - - enum { allow_multimapping = Allow_multimapping }; - - concurrent_unordered_map_traits() : my_hash_compare() {} - concurrent_unordered_map_traits(const hash_compare& hc) : my_hash_compare(hc) {} - - template - static const Key& get_key(const std::pair& value) { - return (value.first); - } - - hash_compare my_hash_compare; // the comparator predicate for keys -}; - -template -class concurrent_unordered_multimap; - -template , typename Key_equality = std::equal_to, - typename Allocator = tbb::tbb_allocator > > -class concurrent_unordered_map : - public internal::concurrent_unordered_base< concurrent_unordered_map_traits, Allocator, false> > -{ - // Base type definitions - typedef internal::hash_compare hash_compare; - typedef concurrent_unordered_map_traits traits_type; - typedef internal::concurrent_unordered_base< traits_type > base_type; -#if __TBB_EXTRA_DEBUG -public: -#endif - using traits_type::allow_multimapping; -public: - using base_type::end; - using base_type::find; - using base_type::insert; - - // Type definitions - typedef Key key_type; - typedef typename base_type::value_type value_type; - typedef T mapped_type; - typedef Hasher hasher; - typedef Key_equality key_equal; - typedef hash_compare key_compare; - - typedef typename base_type::allocator_type allocator_type; - typedef typename base_type::pointer pointer; - typedef typename base_type::const_pointer const_pointer; - typedef typename base_type::reference reference; - typedef typename base_type::const_reference const_reference; - - typedef typename base_type::size_type size_type; - typedef typename base_type::difference_type difference_type; - - typedef typename base_type::iterator iterator; - typedef typename base_type::const_iterator const_iterator; - typedef typename base_type::iterator local_iterator; - typedef typename base_type::const_iterator const_local_iterator; -#if __TBB_UNORDERED_NODE_HANDLE_PRESENT - typedef typename base_type::node_type node_type; -#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT - - // Construction/destruction/copying - explicit concurrent_unordered_map(size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - {} - - concurrent_unordered_map(size_type n_of_buckets, const allocator_type& a) - : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) - {} - - concurrent_unordered_map(size_type n_of_buckets, const hasher& a_hasher, const allocator_type& a) - : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) - {} - - explicit concurrent_unordered_map(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) - {} - - template - concurrent_unordered_map(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - { - insert(first, last); - } - - template - concurrent_unordered_map(Iterator first, Iterator last, size_type n_of_buckets, const allocator_type& a) - : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) - { - insert(first, last); - } - - template - concurrent_unordered_map(Iterator first, Iterator last, size_type n_of_buckets, const hasher& a_hasher, - const allocator_type& a) - : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) - { - insert(first, last); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from initializer_list - concurrent_unordered_map(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - { - insert(il.begin(),il.end()); - } - - concurrent_unordered_map(std::initializer_list il, size_type n_of_buckets, const allocator_type& a) - : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) - { - insert(il.begin(), il.end()); - } - - concurrent_unordered_map(std::initializer_list il, size_type n_of_buckets, const hasher& a_hasher, - const allocator_type& a) - : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) - { - insert(il.begin(), il.end()); - } - -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - - -#if __TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT - concurrent_unordered_map(const concurrent_unordered_map& table) - : base_type(table) - {} - - concurrent_unordered_map& operator=(const concurrent_unordered_map& table) - { - return static_cast(base_type::operator=(table)); - } - - concurrent_unordered_map(concurrent_unordered_map&& table) - : base_type(std::move(table)) - {} - - concurrent_unordered_map& operator=(concurrent_unordered_map&& table) - { - return static_cast(base_type::operator=(std::move(table))); - } -#endif //__TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT - -#if __TBB_CPP11_RVALUE_REF_PRESENT - concurrent_unordered_map(concurrent_unordered_map&& table, const Allocator& a) : base_type(std::move(table), a) - {} -#endif /*__TBB_CPP11_RVALUE_REF_PRESENT*/ - -#if __TBB_UNORDERED_NODE_HANDLE_PRESENT - template - void merge(concurrent_unordered_map& source) - { this->internal_merge(source); } - - template - void merge(concurrent_unordered_map&& source) - { this->internal_merge(source); } - - template - void merge(concurrent_unordered_multimap& source) - { this->internal_merge(source); } - - template - void merge(concurrent_unordered_multimap&& source) - { this->internal_merge(source); } - -#endif //__TBB_UNORDERED_NODE_HANDLE_PRESENT - - concurrent_unordered_map(const concurrent_unordered_map& table, const Allocator& a) - : base_type(table, a) - {} - - // Observers - mapped_type& operator[](const key_type& key) - { - iterator where = find(key); - - if (where == end()) - { - where = insert(std::pair(key, mapped_type())).first; - } - - return ((*where).second); - } - - mapped_type& at(const key_type& key) - { - iterator where = find(key); - - if (where == end()) - { - tbb::internal::throw_exception(tbb::internal::eid_invalid_key); - } - - return ((*where).second); - } - - const mapped_type& at(const key_type& key) const - { - const_iterator where = find(key); - - if (where == end()) - { - tbb::internal::throw_exception(tbb::internal::eid_invalid_key); - } - - return ((*where).second); - } -}; - -#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT - -namespace internal { -using namespace tbb::internal; - -template typename Map, typename Key, typename Element, typename... Args> -using cu_map_t = Map< - Key, Element, - std::conditional_t< (sizeof...(Args)>0) && !is_allocator_v< pack_element_t<0, Args...> >, - pack_element_t<0, Args...>, tbb_hash >, - std::conditional_t< (sizeof...(Args)>1) && !is_allocator_v< pack_element_t<1, Args...> >, - pack_element_t<1, Args...>, std::equal_to >, - std::conditional_t< (sizeof...(Args)>0) && is_allocator_v< pack_element_t >, - pack_element_t, tbb_allocator > > ->; -} - -// Deduction guide for the constructor from two iterators -template -concurrent_unordered_map (I, I) --> internal::cu_map_t, internal::iterator_mapped_t>; - -// Deduction guide for the constructor from two iterators and hasher/equality/allocator -template -concurrent_unordered_map(I, I, size_t, Args...) --> internal::cu_map_t, internal::iterator_mapped_t, Args...>; - -// Deduction guide for the constructor from an initializer_list -template -concurrent_unordered_map(std::initializer_list>) --> internal::cu_map_t; - -// Deduction guide for the constructor from an initializer_list and hasher/equality/allocator -template -concurrent_unordered_map(std::initializer_list>, size_t, Args...) --> internal::cu_map_t; - -#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ - -template < typename Key, typename T, typename Hasher = tbb::tbb_hash, typename Key_equality = std::equal_to, - typename Allocator = tbb::tbb_allocator > > -class concurrent_unordered_multimap : - public internal::concurrent_unordered_base< concurrent_unordered_map_traits< Key, T, - internal::hash_compare, Allocator, true> > -{ - // Base type definitions - typedef internal::hash_compare hash_compare; - typedef concurrent_unordered_map_traits traits_type; - typedef internal::concurrent_unordered_base base_type; -#if __TBB_EXTRA_DEBUG -public: -#endif - using traits_type::allow_multimapping; -public: - using base_type::insert; - - // Type definitions - typedef Key key_type; - typedef typename base_type::value_type value_type; - typedef T mapped_type; - typedef Hasher hasher; - typedef Key_equality key_equal; - typedef hash_compare key_compare; - - typedef typename base_type::allocator_type allocator_type; - typedef typename base_type::pointer pointer; - typedef typename base_type::const_pointer const_pointer; - typedef typename base_type::reference reference; - typedef typename base_type::const_reference const_reference; - - typedef typename base_type::size_type size_type; - typedef typename base_type::difference_type difference_type; - - typedef typename base_type::iterator iterator; - typedef typename base_type::const_iterator const_iterator; - typedef typename base_type::iterator local_iterator; - typedef typename base_type::const_iterator const_local_iterator; -#if __TBB_UNORDERED_NODE_HANDLE_PRESENT - typedef typename base_type::node_type node_type; -#endif //__TBB_UNORDERED_NODE_HANDLE_PRESENT - - // Construction/destruction/copying - explicit concurrent_unordered_multimap(size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - {} - - concurrent_unordered_multimap(size_type n_of_buckets, const allocator_type& a) - : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) - {} - - concurrent_unordered_multimap(size_type n_of_buckets, const hasher& a_hasher, const allocator_type& a) - : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) - {} - - explicit concurrent_unordered_multimap(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) - {} - - template - concurrent_unordered_multimap(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets,key_compare(a_hasher,a_keyeq), a) - { - insert(first, last); - } - - template - concurrent_unordered_multimap(Iterator first, Iterator last, size_type n_of_buckets, const allocator_type& a) - : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) - { - insert(first, last); - } - - template - concurrent_unordered_multimap(Iterator first, Iterator last, size_type n_of_buckets, const hasher& a_hasher, - const allocator_type& a) - : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) - { - insert(first, last); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from initializer_list - concurrent_unordered_multimap(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - { - insert(il.begin(),il.end()); - } - - concurrent_unordered_multimap(std::initializer_list il, size_type n_of_buckets, const allocator_type& a) - : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) - { - insert(il.begin(), il.end()); - } - - concurrent_unordered_multimap(std::initializer_list il, size_type n_of_buckets, const hasher& a_hasher, - const allocator_type& a) - : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) - { - insert(il.begin(), il.end()); - } - -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - -#if __TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT - concurrent_unordered_multimap(const concurrent_unordered_multimap& table) - : base_type(table) - {} - - concurrent_unordered_multimap& operator=(const concurrent_unordered_multimap& table) - { - return static_cast(base_type::operator=(table)); - } - - concurrent_unordered_multimap(concurrent_unordered_multimap&& table) - : base_type(std::move(table)) - {} - - concurrent_unordered_multimap& operator=(concurrent_unordered_multimap&& table) - { - return static_cast(base_type::operator=(std::move(table))); - } -#endif //__TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT - -#if __TBB_CPP11_RVALUE_REF_PRESENT - concurrent_unordered_multimap(concurrent_unordered_multimap&& table, const Allocator& a) : base_type(std::move(table), a) - {} -#endif /*__TBB_CPP11_RVALUE_REF_PRESENT*/ - -#if __TBB_UNORDERED_NODE_HANDLE_PRESENT - template - void merge(concurrent_unordered_map& source) - { this->internal_merge(source); } - - template - void merge(concurrent_unordered_map&& source) - { this->internal_merge(source); } - - template - void merge(concurrent_unordered_multimap& source) - { this->internal_merge(source); } - - template - void merge(concurrent_unordered_multimap&& source) - { this->internal_merge(source); } - -#endif //__TBB_UNORDERED_NODE_HANDLE_PRESENT - - concurrent_unordered_multimap(const concurrent_unordered_multimap& table, const Allocator& a) - : base_type(table, a) - {} -}; - -#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT - -// Deduction guide for the constructor from two iterators -template -concurrent_unordered_multimap (I, I) --> internal::cu_map_t, internal::iterator_mapped_t>; - -// Deduction guide for the constructor from two iterators and hasher/equality/allocator -template -concurrent_unordered_multimap(I, I, size_t, Args...) --> internal::cu_map_t, internal::iterator_mapped_t, Args...>; - -// Deduction guide for the constructor from an initializer_list -template -concurrent_unordered_multimap(std::initializer_list>) --> internal::cu_map_t; - -// Deduction guide for the constructor from an initializer_list and hasher/equality/allocator -template -concurrent_unordered_multimap(std::initializer_list>, size_t, Args...) --> internal::cu_map_t; - -#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ -} // namespace interface5 - -using interface5::concurrent_unordered_map; -using interface5::concurrent_unordered_multimap; - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_concurrent_unordered_map_H_include_area - -#endif// __TBB_concurrent_unordered_map_H +#include "../oneapi/tbb/concurrent_unordered_map.h" diff --git a/src/3rd party/tbb/concurrent_unordered_set.h b/src/3rd party/tbb/concurrent_unordered_set.h index 751488dc94..81a8f9c37d 100644 --- a/src/3rd party/tbb/concurrent_unordered_set.h +++ b/src/3rd party/tbb/concurrent_unordered_set.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,435 +14,4 @@ limitations under the License. */ -/* Container implementations in this header are based on PPL implementations - provided by Microsoft. */ - -#ifndef __TBB_concurrent_unordered_set_H -#define __TBB_concurrent_unordered_set_H - -#define __TBB_concurrent_unordered_set_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "internal/_concurrent_unordered_impl.h" - -namespace tbb -{ - -namespace interface5 { - -// Template class for hash set traits -template -class concurrent_unordered_set_traits -{ -protected: - typedef Key value_type; - typedef Key key_type; - typedef Hash_compare hash_compare; - typedef typename tbb::internal::allocator_rebind::type allocator_type; -#if __TBB_UNORDERED_NODE_HANDLE_PRESENT - typedef tbb::internal::node_handle::node, - allocator_type> node_type; -#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT - - enum { allow_multimapping = Allow_multimapping }; - - concurrent_unordered_set_traits() : my_hash_compare() {} - concurrent_unordered_set_traits(const hash_compare& hc) : my_hash_compare(hc) {} - - static const Key& get_key(const value_type& value) { - return value; - } - - hash_compare my_hash_compare; // the comparator predicate for keys -}; - -template -class concurrent_unordered_multiset; - -template , typename Key_equality = std::equal_to, typename Allocator = tbb::tbb_allocator > -class concurrent_unordered_set : public internal::concurrent_unordered_base< concurrent_unordered_set_traits, Allocator, false> > -{ - // Base type definitions - typedef internal::hash_compare hash_compare; - typedef concurrent_unordered_set_traits traits_type; - typedef internal::concurrent_unordered_base< traits_type > base_type; -#if __TBB_EXTRA_DEBUG -public: -#endif - using traits_type::allow_multimapping; -public: - using base_type::insert; - - // Type definitions - typedef Key key_type; - typedef typename base_type::value_type value_type; - typedef Key mapped_type; - typedef Hasher hasher; - typedef Key_equality key_equal; - typedef hash_compare key_compare; - - typedef typename base_type::allocator_type allocator_type; - typedef typename base_type::pointer pointer; - typedef typename base_type::const_pointer const_pointer; - typedef typename base_type::reference reference; - typedef typename base_type::const_reference const_reference; - - typedef typename base_type::size_type size_type; - typedef typename base_type::difference_type difference_type; - - typedef typename base_type::iterator iterator; - typedef typename base_type::const_iterator const_iterator; - typedef typename base_type::iterator local_iterator; - typedef typename base_type::const_iterator const_local_iterator; -#if __TBB_UNORDERED_NODE_HANDLE_PRESENT - typedef typename base_type::node_type node_type; -#endif /*__TBB_UNORDERED_NODE_HANDLE_PRESENT*/ - - // Construction/destruction/copying - explicit concurrent_unordered_set(size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), - const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - {} - - concurrent_unordered_set(size_type n_of_buckets, const allocator_type& a) - : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) - {} - - concurrent_unordered_set(size_type n_of_buckets, const hasher& a_hasher, const allocator_type& a) - : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) - {} - - explicit concurrent_unordered_set(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) - {} - - template - concurrent_unordered_set(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - { - insert(first, last); - } - - template - concurrent_unordered_set(Iterator first, Iterator last, size_type n_of_buckets, const allocator_type& a) - : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) - { - insert(first, last); - } - - template - concurrent_unordered_set(Iterator first, Iterator last, size_type n_of_buckets, const hasher& a_hasher, const allocator_type& a) - : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) - { - insert(first, last); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from initializer_list - concurrent_unordered_set(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, const hasher& a_hasher = hasher(), - const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - { - insert(il.begin(),il.end()); - } - - concurrent_unordered_set(std::initializer_list il, size_type n_of_buckets, const allocator_type& a) - : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) - { - insert(il.begin(), il.end()); - } - - concurrent_unordered_set(std::initializer_list il, size_type n_of_buckets, const hasher& a_hasher, const allocator_type& a) - : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) - { - insert(il.begin(), il.end()); - } - -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - -#if __TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT - concurrent_unordered_set(const concurrent_unordered_set& table) - : base_type(table) - {} - - concurrent_unordered_set& operator=(const concurrent_unordered_set& table) - { - return static_cast(base_type::operator=(table)); - } - - concurrent_unordered_set(concurrent_unordered_set&& table) - : base_type(std::move(table)) - {} - - concurrent_unordered_set& operator=(concurrent_unordered_set&& table) - { - return static_cast(base_type::operator=(std::move(table))); - } -#endif //__TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT - -#if __TBB_CPP11_RVALUE_REF_PRESENT - concurrent_unordered_set(concurrent_unordered_set&& table, const Allocator& a) - : base_type(std::move(table), a) - {} -#endif /*__TBB_CPP11_RVALUE_REF_PRESENT*/ - -#if __TBB_UNORDERED_NODE_HANDLE_PRESENT - template - void merge(concurrent_unordered_set& source) - { this->internal_merge(source); } - - template - void merge(concurrent_unordered_set&& source) - { this->internal_merge(source); } - - template - void merge(concurrent_unordered_multiset& source) - { this->internal_merge(source); } - - template - void merge(concurrent_unordered_multiset&& source) - { this->internal_merge(source); } - -#endif //__TBB_UNORDERED_NODE_HANDLE_PRESENT - - concurrent_unordered_set(const concurrent_unordered_set& table, const Allocator& a) - : base_type(table, a) - {} - -}; - -#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT - -namespace internal { -using namespace tbb::internal; - -template typename Set, typename T, typename... Args> -using cu_set_t = Set < - T, - std::conditional_t< (sizeof...(Args)>0) && !is_allocator_v< pack_element_t<0, Args...> >, - pack_element_t<0, Args...>, tbb_hash >, - std::conditional_t< (sizeof...(Args)>1) && !is_allocator_v< pack_element_t<1, Args...> >, - pack_element_t<1, Args...>, std::equal_to >, - std::conditional_t< (sizeof...(Args)>0) && is_allocator_v< pack_element_t >, - pack_element_t, tbb_allocator > ->; -} - -// Deduction guide for the constructor from two iterators -template -concurrent_unordered_set(I, I) --> internal::cu_set_t>; - -// Deduction guide for the constructor from two iterators and hasher/equality/allocator -template -concurrent_unordered_set(I, I, size_t, Args...) --> internal::cu_set_t, Args...>; - -// Deduction guide for the constructor from an initializer_list -template -concurrent_unordered_set(std::initializer_list) --> internal::cu_set_t; - -// Deduction guide for the constructor from an initializer_list and hasher/equality/allocator -template -concurrent_unordered_set(std::initializer_list, size_t, Args...) --> internal::cu_set_t; - -#endif /*__TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ - -template , typename Key_equality = std::equal_to, - typename Allocator = tbb::tbb_allocator > -class concurrent_unordered_multiset : - public internal::concurrent_unordered_base< concurrent_unordered_set_traits, Allocator, true> > -{ - // Base type definitions - typedef internal::hash_compare hash_compare; - typedef concurrent_unordered_set_traits traits_type; - typedef internal::concurrent_unordered_base< traits_type > base_type; -#if __TBB_EXTRA_DEBUG -public: -#endif - using traits_type::allow_multimapping; -public: - using base_type::insert; - - // Type definitions - typedef Key key_type; - typedef typename base_type::value_type value_type; - typedef Key mapped_type; - typedef Hasher hasher; - typedef Key_equality key_equal; - typedef hash_compare key_compare; - - typedef typename base_type::allocator_type allocator_type; - typedef typename base_type::pointer pointer; - typedef typename base_type::const_pointer const_pointer; - typedef typename base_type::reference reference; - typedef typename base_type::const_reference const_reference; - - typedef typename base_type::size_type size_type; - typedef typename base_type::difference_type difference_type; - - typedef typename base_type::iterator iterator; - typedef typename base_type::const_iterator const_iterator; - typedef typename base_type::iterator local_iterator; - typedef typename base_type::const_iterator const_local_iterator; -#if __TBB_UNORDERED_NODE_HANDLE_PRESENT - typedef typename base_type::node_type node_type; -#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT - - // Construction/destruction/copying - explicit concurrent_unordered_multiset(size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - {} - - concurrent_unordered_multiset(size_type n_of_buckets, const allocator_type& a) - : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) - {} - - concurrent_unordered_multiset(size_type n_of_buckets, const hasher& a_hasher, - const allocator_type& a) - : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) - {} - - explicit concurrent_unordered_multiset(const Allocator& a) : base_type(base_type::initial_bucket_number, key_compare(), a) - {} - - template - concurrent_unordered_multiset(Iterator first, Iterator last, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), - const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - { - insert(first, last); - } - - template - concurrent_unordered_multiset(Iterator first, Iterator last, size_type n_of_buckets, const allocator_type& a) - : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) - { - insert(first, last); - } - - template - concurrent_unordered_multiset(Iterator first, Iterator last, size_type n_of_buckets, const hasher& a_hasher, - const allocator_type& a) - : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) - { - insert(first, last); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from initializer_list - concurrent_unordered_multiset(std::initializer_list il, size_type n_of_buckets = base_type::initial_bucket_number, - const hasher& a_hasher = hasher(), const key_equal& a_keyeq = key_equal(), const allocator_type& a = allocator_type()) - : base_type(n_of_buckets, key_compare(a_hasher, a_keyeq), a) - { - insert(il.begin(),il.end()); - } - - concurrent_unordered_multiset(std::initializer_list il, size_type n_of_buckets, const allocator_type& a) - : base_type(n_of_buckets, key_compare(hasher(), key_equal()), a) - { - insert(il.begin(), il.end()); - } - - concurrent_unordered_multiset(std::initializer_list il, size_type n_of_buckets, const hasher& a_hasher, - const allocator_type& a) - : base_type(n_of_buckets, key_compare(a_hasher, key_equal()), a) - { - insert(il.begin(), il.end()); - } - -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - - -#if __TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT - concurrent_unordered_multiset(const concurrent_unordered_multiset& table) - : base_type(table) - {} - - concurrent_unordered_multiset& operator=(const concurrent_unordered_multiset& table) - { - return static_cast(base_type::operator=(table)); - } - - concurrent_unordered_multiset(concurrent_unordered_multiset&& table) - : base_type(std::move(table)) - {} - - concurrent_unordered_multiset& operator=(concurrent_unordered_multiset&& table) - { - return static_cast(base_type::operator=(std::move(table))); - } -#endif //__TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_MOVE_PRESENT - -#if __TBB_CPP11_RVALUE_REF_PRESENT - concurrent_unordered_multiset(concurrent_unordered_multiset&& table, const Allocator& a) - : base_type(std::move(table), a) - { - } -#endif /*__TBB_CPP11_RVALUE_REF_PRESENT*/ - -#if __TBB_UNORDERED_NODE_HANDLE_PRESENT - template - void merge(concurrent_unordered_set& source) - { this->internal_merge(source); } - - template - void merge(concurrent_unordered_set&& source) - { this->internal_merge(source); } - - template - void merge(concurrent_unordered_multiset& source) - { this->internal_merge(source); } - - template - void merge(concurrent_unordered_multiset&& source) - { this->internal_merge(source); } - -#endif //__TBB_UNORDERED_NODE_HANDLE_PRESENT - - concurrent_unordered_multiset(const concurrent_unordered_multiset& table, const Allocator& a) - : base_type(table, a) - {} -}; - -#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT - -// Deduction guide for the constructor from two iterators -template -concurrent_unordered_multiset(I, I) --> internal::cu_set_t>; - -// Deduction guide for the constructor from two iterators and hasher/equality/allocator -template -concurrent_unordered_multiset(I, I, size_t, Args...) --> internal::cu_set_t, Args...>; - -// Deduction guide for the constructor from an initializer_list -template -concurrent_unordered_multiset(std::initializer_list) --> internal::cu_set_t; - -// Deduction guide for the constructor from an initializer_list and hasher/equality/allocator -template -concurrent_unordered_multiset(std::initializer_list, size_t, Args...) --> internal::cu_set_t; - -#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ -} // namespace interface5 - -using interface5::concurrent_unordered_set; -using interface5::concurrent_unordered_multiset; - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_concurrent_unordered_set_H_include_area - -#endif// __TBB_concurrent_unordered_set_H +#include "../oneapi/tbb/concurrent_unordered_set.h" diff --git a/src/3rd party/tbb/concurrent_vector.h b/src/3rd party/tbb/concurrent_vector.h index 5ddc38fbe9..c1fc97c623 100644 --- a/src/3rd party/tbb/concurrent_vector.h +++ b/src/3rd party/tbb/concurrent_vector.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,1383 +14,4 @@ limitations under the License. */ -#ifndef __TBB_concurrent_vector_H -#define __TBB_concurrent_vector_H - -#define __TBB_concurrent_vector_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "tbb_stddef.h" -#include "tbb_exception.h" -#include "atomic.h" -#include "cache_aligned_allocator.h" -#include "blocked_range.h" -#include "tbb_machine.h" -#include "tbb_profiling.h" -#include -#include // for memset() -#include __TBB_STD_SWAP_HEADER -#include -#include - -#include "internal/_allocator_traits.h" - -#if _MSC_VER==1500 && !__INTEL_COMPILER - // VS2008/VC9 seems to have an issue; limits pull in math.h - #pragma warning( push ) - #pragma warning( disable: 4985 ) -#endif -#include /* std::numeric_limits */ -#if _MSC_VER==1500 && !__INTEL_COMPILER - #pragma warning( pop ) -#endif - -#if __TBB_INITIALIZER_LISTS_PRESENT - #include -#endif - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - // Workaround for overzealous compiler warnings in /Wp64 mode - #pragma warning (push) -#if defined(_Wp64) - #pragma warning (disable: 4267) -#endif - #pragma warning (disable: 4127) //warning C4127: conditional expression is constant -#endif - -namespace tbb { - -template > -class concurrent_vector; - -//! @cond INTERNAL -namespace internal { - - template - class vector_iterator; - - //! Bad allocation marker - static void *const vector_allocation_error_flag = reinterpret_cast(size_t(63)); - - //! Exception helper function - template - void handle_unconstructed_elements(T* array, size_t n_of_elements){ - std::memset( static_cast(array), 0, n_of_elements * sizeof( T ) ); - } - - //! Base class of concurrent vector implementation. - /** @ingroup containers */ - class concurrent_vector_base_v3 { - protected: - - // Basic types declarations - typedef size_t segment_index_t; - typedef size_t size_type; - - // Using enumerations due to Mac linking problems of static const variables - enum { - // Size constants - default_initial_segments = 1, // 2 initial items - //! Number of slots for segment pointers inside the class - pointers_per_short_table = 3, // to fit into 8 words of entire structure - pointers_per_long_table = sizeof(segment_index_t) * 8 // one segment per bit - }; - - struct segment_not_used {}; - struct segment_allocated {}; - struct segment_allocation_failed {}; - - class segment_t; - class segment_value_t { - void* array; - private: - //TODO: More elegant way to grant access to selected functions _only_? - friend class segment_t; - explicit segment_value_t(void* an_array):array(an_array) {} - public: - friend bool operator==(segment_value_t const& lhs, segment_not_used ) { return lhs.array == 0;} - friend bool operator==(segment_value_t const& lhs, segment_allocated) { return lhs.array > internal::vector_allocation_error_flag;} - friend bool operator==(segment_value_t const& lhs, segment_allocation_failed) { return lhs.array == internal::vector_allocation_error_flag;} - template - friend bool operator!=(segment_value_t const& lhs, argument_type arg) { return ! (lhs == arg);} - - template - T* pointer() const { return static_cast(const_cast(array)); } - }; - - friend void enforce_segment_allocated(segment_value_t const& s, internal::exception_id exception = eid_bad_last_alloc){ - if(s != segment_allocated()){ - internal::throw_exception(exception); - } - } - - // Segment pointer. - class segment_t { - atomic array; - public: - segment_t(){ store(segment_not_used());} - //Copy ctor and assignment operator are defined to ease using of stl algorithms. - //These algorithms usually not a synchronization point, so, semantic is - //intentionally relaxed here. - segment_t(segment_t const& rhs ){ array.store(rhs.array.load());} - - void swap(segment_t & rhs ){ - tbb::internal::swap(array, rhs.array); - } - - segment_t& operator=(segment_t const& rhs ){ - array.store(rhs.array.load()); - return *this; - } - - template - segment_value_t load() const { return segment_value_t(array.load());} - - template - void store(segment_not_used) { - array.store(0); - } - - template - void store(segment_allocation_failed) { - __TBB_ASSERT(load() != segment_allocated(),"transition from \"allocated\" to \"allocation failed\" state looks non-logical"); - array.store(internal::vector_allocation_error_flag); - } - - template - void store(void* allocated_segment_pointer) __TBB_NOEXCEPT(true) { - __TBB_ASSERT(segment_value_t(allocated_segment_pointer) == segment_allocated(), - "other overloads of store should be used for marking segment as not_used or allocation_failed" ); - array.store(allocated_segment_pointer); - } - -#if TBB_USE_ASSERT - ~segment_t() { - __TBB_ASSERT(load() != segment_allocated(), "should have been freed by clear" ); - } -#endif /* TBB_USE_ASSERT */ - }; - friend void swap(segment_t & , segment_t & ) __TBB_NOEXCEPT(true); - - // Data fields - - //! allocator function pointer - void* (*vector_allocator_ptr)(concurrent_vector_base_v3 &, size_t); - - //! count of segments in the first block - atomic my_first_block; - - //! Requested size of vector - atomic my_early_size; - - //! Pointer to the segments table - atomic my_segment; - - //! embedded storage of segment pointers - segment_t my_storage[pointers_per_short_table]; - - // Methods - - concurrent_vector_base_v3() { - //Here the semantic is intentionally relaxed. - //The reason this is next: - //Object that is in middle of construction (i.e. its constructor is not yet finished) - //cannot be used concurrently until the construction is finished. - //Thus to flag other threads that construction is finished, some synchronization with - //acquire-release semantic should be done by the (external) code that uses the vector. - //So, no need to do the synchronization inside the vector. - - my_early_size.store(0); - my_first_block.store(0); // here is not default_initial_segments - my_segment.store(my_storage); - } - - __TBB_EXPORTED_METHOD ~concurrent_vector_base_v3(); - - //these helpers methods use the fact that segments are allocated so - //that every segment size is a (increasing) power of 2. - //with one exception 0 segment has size of 2 as well segment 1; - //e.g. size of segment with index of 3 is 2^3=8; - static segment_index_t segment_index_of( size_type index ) { - return segment_index_t( __TBB_Log2( index|1 ) ); - } - - static segment_index_t segment_base( segment_index_t k ) { - return (segment_index_t(1)< - friend class vector_iterator; - - }; - - inline void swap(concurrent_vector_base_v3::segment_t & lhs, concurrent_vector_base_v3::segment_t & rhs) __TBB_NOEXCEPT(true) { - lhs.swap(rhs); - } - - typedef concurrent_vector_base_v3 concurrent_vector_base; - - //! Meets requirements of a forward iterator for STL and a Value for a blocked_range.*/ - /** Value is either the T or const T type of the container. - @ingroup containers */ - template - class vector_iterator - { - //! concurrent_vector over which we are iterating. - Container* my_vector; - - //! Index into the vector - size_t my_index; - - //! Caches my_vector->internal_subscript(my_index) - /** NULL if cached value is not available */ - mutable Value* my_item; - - template - friend vector_iterator operator+( ptrdiff_t offset, const vector_iterator& v ); - - template - friend bool operator==( const vector_iterator& i, const vector_iterator& j ); - - template - friend bool operator<( const vector_iterator& i, const vector_iterator& j ); - - template - friend ptrdiff_t operator-( const vector_iterator& i, const vector_iterator& j ); - - template - friend class internal::vector_iterator; - -#if !__TBB_TEMPLATE_FRIENDS_BROKEN - template - friend class tbb::concurrent_vector; -#else -public: -#endif - - vector_iterator( const Container& vector, size_t index, void *ptr = 0 ) : - my_vector(const_cast(&vector)), - my_index(index), - my_item(static_cast(ptr)) - {} - - public: - //! Default constructor - vector_iterator() : my_vector(NULL), my_index(~size_t(0)), my_item(NULL) {} - - vector_iterator( const vector_iterator& other ) : - my_vector(other.my_vector), - my_index(other.my_index), - my_item(other.my_item) - {} - - vector_iterator& operator=( const vector_iterator& other ) - { - my_vector=other.my_vector; - my_index=other.my_index; - my_item=other.my_item; - return *this; - } - - vector_iterator operator+( ptrdiff_t offset ) const { - return vector_iterator( *my_vector, my_index+offset ); - } - vector_iterator &operator+=( ptrdiff_t offset ) { - my_index+=offset; - my_item = NULL; - return *this; - } - vector_iterator operator-( ptrdiff_t offset ) const { - return vector_iterator( *my_vector, my_index-offset ); - } - vector_iterator &operator-=( ptrdiff_t offset ) { - my_index-=offset; - my_item = NULL; - return *this; - } - Value& operator*() const { - Value* item = my_item; - if( !item ) { - item = my_item = &my_vector->internal_subscript(my_index); - } - __TBB_ASSERT( item==&my_vector->internal_subscript(my_index), "corrupt cache" ); - return *item; - } - Value& operator[]( ptrdiff_t k ) const { - return my_vector->internal_subscript(my_index+k); - } - Value* operator->() const {return &operator*();} - - //! Pre increment - vector_iterator& operator++() { - size_t element_index = ++my_index; - if( my_item ) { - //TODO: consider using of knowledge about "first_block optimization" here as well? - if( concurrent_vector_base::is_first_element_in_segment(element_index)) { - //if the iterator crosses a segment boundary, the pointer become invalid - //as possibly next segment is in another memory location - my_item= NULL; - } else { - ++my_item; - } - } - return *this; - } - - //! Pre decrement - vector_iterator& operator--() { - __TBB_ASSERT( my_index>0, "operator--() applied to iterator already at beginning of concurrent_vector" ); - size_t element_index = my_index--; - if( my_item ) { - if(concurrent_vector_base::is_first_element_in_segment(element_index)) { - //if the iterator crosses a segment boundary, the pointer become invalid - //as possibly next segment is in another memory location - my_item= NULL; - } else { - --my_item; - } - } - return *this; - } - - //! Post increment - vector_iterator operator++(int) { - vector_iterator result = *this; - operator++(); - return result; - } - - //! Post decrement - vector_iterator operator--(int) { - vector_iterator result = *this; - operator--(); - return result; - } - - // STL support - - typedef ptrdiff_t difference_type; - typedef Value value_type; - typedef Value* pointer; - typedef Value& reference; - typedef std::random_access_iterator_tag iterator_category; - }; - - template - vector_iterator operator+( ptrdiff_t offset, const vector_iterator& v ) { - return vector_iterator( *v.my_vector, v.my_index+offset ); - } - - template - bool operator==( const vector_iterator& i, const vector_iterator& j ) { - return i.my_index==j.my_index && i.my_vector == j.my_vector; - } - - template - bool operator!=( const vector_iterator& i, const vector_iterator& j ) { - return !(i==j); - } - - template - bool operator<( const vector_iterator& i, const vector_iterator& j ) { - return i.my_index - bool operator>( const vector_iterator& i, const vector_iterator& j ) { - return j - bool operator>=( const vector_iterator& i, const vector_iterator& j ) { - return !(i - bool operator<=( const vector_iterator& i, const vector_iterator& j ) { - return !(j - ptrdiff_t operator-( const vector_iterator& i, const vector_iterator& j ) { - return ptrdiff_t(i.my_index)-ptrdiff_t(j.my_index); - } - - template - class allocator_base { - public: - typedef typename tbb::internal::allocator_rebind::type allocator_type; - allocator_type my_allocator; - allocator_base(const allocator_type &a = allocator_type() ) : my_allocator(a) {} - }; - -} // namespace internal -//! @endcond - -//! Concurrent vector container -/** concurrent_vector is a container having the following main properties: - - It provides random indexed access to its elements. The index of the first element is 0. - - It ensures safe concurrent growing its size (different threads can safely append new elements). - - Adding new elements does not invalidate existing iterators and does not change indices of existing items. - -@par Compatibility - The class meets all Container Requirements and Reversible Container Requirements from - C++ Standard (See ISO/IEC 14882:2003(E), clause 23.1). But it doesn't meet - Sequence Requirements due to absence of insert() and erase() methods. - -@par Exception Safety - Methods working with memory allocation and/or new elements construction can throw an - exception if allocator fails to allocate memory or element's default constructor throws one. - Concurrent vector's element of type T must conform to the following requirements: - - Throwing an exception is forbidden for destructor of T. - - Default constructor of T must not throw an exception OR its non-virtual destructor must safely work when its object memory is zero-initialized. - . - Otherwise, the program's behavior is undefined. -@par - If an exception happens inside growth or assignment operation, an instance of the vector becomes invalid unless it is stated otherwise in the method documentation. - Invalid state means: - - There are no guarantees that all items were initialized by a constructor. The rest of items is zero-filled, including item where exception happens. - - An invalid vector instance cannot be repaired; it is unable to grow anymore. - - Size and capacity reported by the vector are incorrect, and calculated as if the failed operation were successful. - - Attempt to access not allocated elements using operator[] or iterators results in access violation or segmentation fault exception, and in case of using at() method a C++ exception is thrown. - . - If a concurrent grow operation successfully completes, all the elements it has added to the vector will remain valid and accessible even if one of subsequent grow operations fails. - -@par Fragmentation - Unlike an STL vector, a concurrent_vector does not move existing elements if it needs - to allocate more memory. The container is divided into a series of contiguous arrays of - elements. The first reservation, growth, or assignment operation determines the size of - the first array. Using small number of elements as initial size incurs fragmentation that - may increase element access time. Internal layout can be optimized by method compact() that - merges several smaller arrays into one solid. - -@par Changes since TBB 2.1 - - Fixed guarantees of concurrent_vector::size() and grow_to_at_least() methods to assure elements are allocated. - - Methods end()/rbegin()/back() are partly thread-safe since they use size() to get the end of vector - - Added resize() methods (not thread-safe) - - Added cbegin/cend/crbegin/crend methods - - Changed return type of methods grow* and push_back to iterator - -@par Changes since TBB 2.0 - - Implemented exception-safety guarantees - - Added template argument for allocator - - Added allocator argument in constructors - - Faster index calculation - - First growth call specifies a number of segments to be merged in the first allocation. - - Fixed memory blow up for swarm of vector's instances of small size - - Added grow_by(size_type n, const_reference t) growth using copying constructor to init new items. - - Added STL-like constructors. - - Added operators ==, < and derivatives - - Added at() method, approved for using after an exception was thrown inside the vector - - Added get_allocator() method. - - Added assign() methods - - Added compact() method to defragment first segments - - Added swap() method - - range() defaults on grainsize = 1 supporting auto grainsize algorithms. - - @ingroup containers */ -template -class concurrent_vector: protected internal::allocator_base, - private internal::concurrent_vector_base { -private: - template - class generic_range_type: public blocked_range { - public: - typedef T value_type; - typedef T& reference; - typedef const T& const_reference; - typedef I iterator; - typedef ptrdiff_t difference_type; - generic_range_type( I begin_, I end_, size_t grainsize_ = 1) : blocked_range(begin_,end_,grainsize_) {} - template - generic_range_type( const generic_range_type& r) : blocked_range(r.begin(),r.end(),r.grainsize()) {} - generic_range_type( generic_range_type& r, split ) : blocked_range(r,split()) {} - }; - - template - friend class internal::vector_iterator; - -public: - //------------------------------------------------------------------------ - // STL compatible types - //------------------------------------------------------------------------ - typedef internal::concurrent_vector_base_v3::size_type size_type; - typedef typename internal::allocator_base::allocator_type allocator_type; - - typedef T value_type; - typedef ptrdiff_t difference_type; - typedef T& reference; - typedef const T& const_reference; - typedef T *pointer; - typedef const T *const_pointer; - - typedef internal::vector_iterator iterator; - typedef internal::vector_iterator const_iterator; - -#if !defined(_MSC_VER) || _CPPLIB_VER>=300 - // Assume ISO standard definition of std::reverse_iterator - typedef std::reverse_iterator reverse_iterator; - typedef std::reverse_iterator const_reverse_iterator; -#else - // Use non-standard std::reverse_iterator - typedef std::reverse_iterator reverse_iterator; - typedef std::reverse_iterator const_reverse_iterator; -#endif /* defined(_MSC_VER) && (_MSC_VER<1300) */ - - //------------------------------------------------------------------------ - // Parallel algorithm support - //------------------------------------------------------------------------ - typedef generic_range_type range_type; - typedef generic_range_type const_range_type; - - //------------------------------------------------------------------------ - // STL compatible constructors & destructors - //------------------------------------------------------------------------ - - //! Construct empty vector. - explicit concurrent_vector(const allocator_type &a = allocator_type()) - : internal::allocator_base(a), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - } - - //Constructors are not required to have synchronization - //(for more details see comment in the concurrent_vector_base constructor). -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Constructor from initializer_list - concurrent_vector(std::initializer_list init_list, const allocator_type &a = allocator_type()) - : internal::allocator_base(a), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_assign_iterators(init_list.begin(), init_list.end()); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load();; - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load()); - __TBB_RETHROW(); - } - - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - - //! Copying constructor - concurrent_vector( const concurrent_vector& vector, const allocator_type& a = allocator_type() ) - : internal::allocator_base(a), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_copy(vector, sizeof(T), ©_array); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load()); - __TBB_RETHROW(); - } - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move constructor - //TODO add __TBB_NOEXCEPT(true) and static_assert(std::has_nothrow_move_constructor::value) - concurrent_vector( concurrent_vector&& source) - : internal::allocator_base(std::move(source)), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - concurrent_vector_base_v3::internal_swap(source); - } - - concurrent_vector( concurrent_vector&& source, const allocator_type& a) - : internal::allocator_base(a), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - //C++ standard requires instances of an allocator being compared for equality, - //which means that memory allocated by one instance is possible to deallocate with the other one. - if (a == source.my_allocator) { - concurrent_vector_base_v3::internal_swap(source); - } else { - __TBB_TRY { - internal_copy(source, sizeof(T), &move_array); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load()); - __TBB_RETHROW(); - } - } - } - -#endif - - //! Copying constructor for vector with different allocator type - template - concurrent_vector( const concurrent_vector& vector, const allocator_type& a = allocator_type() ) - : internal::allocator_base(a), internal::concurrent_vector_base() - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_copy(vector.internal_vector_base(), sizeof(T), ©_array); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load() ); - __TBB_RETHROW(); - } - } - - //! Construction with initial size specified by argument n - explicit concurrent_vector(size_type n) - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_resize( n, sizeof(T), max_size(), NULL, &destroy_array, &initialize_array ); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load() ); - __TBB_RETHROW(); - } - } - - //! Construction with initial size specified by argument n, initialization by copying of t, and given allocator instance - concurrent_vector(size_type n, const_reference t, const allocator_type& a = allocator_type()) - : internal::allocator_base(a) - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_resize( n, sizeof(T), max_size(), static_cast(&t), &destroy_array, &initialize_array_by ); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load() ); - __TBB_RETHROW(); - } - } - - //! Construction with copying iteration range and given allocator instance - template - concurrent_vector(I first, I last, const allocator_type &a = allocator_type()) - : internal::allocator_base(a) - { - vector_allocator_ptr = &internal_allocator; - __TBB_TRY { - internal_assign_range(first, last, static_cast::is_integer> *>(0) ); - } __TBB_CATCH(...) { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load() ); - __TBB_RETHROW(); - } - } - - //! Assignment - concurrent_vector& operator=( const concurrent_vector& vector ) { - if( this != &vector ) - internal_assign(vector, sizeof(T), &destroy_array, &assign_array, ©_array); - return *this; - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //TODO: add __TBB_NOEXCEPT() - //! Move assignment - concurrent_vector& operator=( concurrent_vector&& other ) { - __TBB_ASSERT(this != &other, "Move assignment to itself is prohibited "); - typedef typename tbb::internal::allocator_traits::propagate_on_container_move_assignment pocma_t; - if(pocma_t::value || this->my_allocator == other.my_allocator) { - concurrent_vector trash (std::move(*this)); - internal_swap(other); - tbb::internal::allocator_move_assignment(this->my_allocator, other.my_allocator, pocma_t()); - } else { - internal_assign(other, sizeof(T), &destroy_array, &move_assign_array, &move_array); - } - return *this; - } -#endif - //TODO: add an template assignment operator? (i.e. with different element type) - - //! Assignment for vector with different allocator type - template - concurrent_vector& operator=( const concurrent_vector& vector ) { - if( static_cast( this ) != static_cast( &vector ) ) - internal_assign(vector.internal_vector_base(), - sizeof(T), &destroy_array, &assign_array, ©_array); - return *this; - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Assignment for initializer_list - concurrent_vector& operator=( std::initializer_list init_list ) { - internal_clear(&destroy_array); - internal_assign_iterators(init_list.begin(), init_list.end()); - return *this; - } -#endif //#if __TBB_INITIALIZER_LISTS_PRESENT - - //------------------------------------------------------------------------ - // Concurrent operations - //------------------------------------------------------------------------ - //! Grow by "delta" elements. - /** Returns iterator pointing to the first new element. */ - iterator grow_by( size_type delta ) { - return iterator(*this, delta ? internal_grow_by( delta, sizeof(T), &initialize_array, NULL ) : my_early_size.load()); - } - - //! Grow by "delta" elements using copying constructor. - /** Returns iterator pointing to the first new element. */ - iterator grow_by( size_type delta, const_reference t ) { - return iterator(*this, delta ? internal_grow_by( delta, sizeof(T), &initialize_array_by, static_cast(&t) ) : my_early_size.load()); - } - - /** Returns iterator pointing to the first new element. */ - template - iterator grow_by( I first, I last ) { - typename std::iterator_traits::difference_type delta = std::distance(first, last); - __TBB_ASSERT( delta >= 0, NULL); - - return iterator(*this, delta ? internal_grow_by(delta, sizeof(T), ©_range, static_cast(&first)) : my_early_size.load()); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - /** Returns iterator pointing to the first new element. */ - iterator grow_by( std::initializer_list init_list ) { - return grow_by( init_list.begin(), init_list.end() ); - } -#endif //#if __TBB_INITIALIZER_LISTS_PRESENT - - //! Append minimal sequence of elements such that size()>=n. - /** The new elements are default constructed. Blocks until all elements in range [0..n) are allocated. - May return while other elements are being constructed by other threads. - Returns iterator that points to beginning of appended sequence. - If no elements were appended, returns iterator pointing to nth element. */ - iterator grow_to_at_least( size_type n ) { - size_type m=0; - if( n ) { - m = internal_grow_to_at_least_with_result( n, sizeof(T), &initialize_array, NULL ); - if( m>n ) m=n; - } - return iterator(*this, m); - }; - - /** Analogous to grow_to_at_least( size_type n ) with exception that the new - elements are initialized by copying of t instead of default construction. */ - iterator grow_to_at_least( size_type n, const_reference t ) { - size_type m=0; - if( n ) { - m = internal_grow_to_at_least_with_result( n, sizeof(T), &initialize_array_by, &t); - if( m>n ) m=n; - } - return iterator(*this, m); - }; - - //! Push item - /** Returns iterator pointing to the new element. */ - iterator push_back( const_reference item ) - { - push_back_helper prolog(*this); - new(prolog.internal_push_back_result()) T(item); - return prolog.return_iterator_and_dismiss(); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Push item, move-aware - /** Returns iterator pointing to the new element. */ - iterator push_back( T&& item ) - { - push_back_helper prolog(*this); - new(prolog.internal_push_back_result()) T(std::move(item)); - return prolog.return_iterator_and_dismiss(); - } -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - //! Push item, create item "in place" with provided arguments - /** Returns iterator pointing to the new element. */ - template - iterator emplace_back( Args&&... args ) - { - push_back_helper prolog(*this); - new(prolog.internal_push_back_result()) T(std::forward(args)...); - return prolog.return_iterator_and_dismiss(); - } -#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif //__TBB_CPP11_RVALUE_REF_PRESENT - //! Get reference to element at given index. - /** This method is thread-safe for concurrent reads, and also while growing the vector, - as long as the calling thread has checked that index < size(). */ - reference operator[]( size_type index ) { - return internal_subscript(index); - } - - //! Get const reference to element at given index. - const_reference operator[]( size_type index ) const { - return internal_subscript(index); - } - - //! Get reference to element at given index. Throws exceptions on errors. - reference at( size_type index ) { - return internal_subscript_with_exceptions(index); - } - - //! Get const reference to element at given index. Throws exceptions on errors. - const_reference at( size_type index ) const { - return internal_subscript_with_exceptions(index); - } - - //! Get range for iterating with parallel algorithms - range_type range( size_t grainsize = 1 ) { - return range_type( begin(), end(), grainsize ); - } - - //! Get const range for iterating with parallel algorithms - const_range_type range( size_t grainsize = 1 ) const { - return const_range_type( begin(), end(), grainsize ); - } - - //------------------------------------------------------------------------ - // Capacity - //------------------------------------------------------------------------ - //! Return size of vector. It may include elements under construction - size_type size() const { - size_type sz = my_early_size, cp = internal_capacity(); - return cp < sz ? cp : sz; - } - - //! Return false if vector is not empty or has elements under construction at least. - bool empty() const {return !my_early_size;} - - //! Maximum size to which array can grow without allocating more memory. Concurrent allocations are not included in the value. - size_type capacity() const {return internal_capacity();} - - //! Allocate enough space to grow to size n without having to allocate more memory later. - /** Like most of the methods provided for STL compatibility, this method is *not* thread safe. - The capacity afterwards may be bigger than the requested reservation. */ - void reserve( size_type n ) { - if( n ) - internal_reserve(n, sizeof(T), max_size()); - } - - //! Resize the vector. Not thread-safe. - void resize( size_type n ) { - internal_resize( n, sizeof(T), max_size(), NULL, &destroy_array, &initialize_array ); - } - - //! Resize the vector, copy t for new elements. Not thread-safe. - void resize( size_type n, const_reference t ) { - internal_resize( n, sizeof(T), max_size(), static_cast(&t), &destroy_array, &initialize_array_by ); - } - - //! Optimize memory usage and fragmentation. - void shrink_to_fit(); - - //! Upper bound on argument to reserve. - size_type max_size() const {return (~size_type(0))/sizeof(T);} - - //------------------------------------------------------------------------ - // STL support - //------------------------------------------------------------------------ - - //! start iterator - iterator begin() {return iterator(*this,0);} - //! end iterator - iterator end() {return iterator(*this,size());} - //! start const iterator - const_iterator begin() const {return const_iterator(*this,0);} - //! end const iterator - const_iterator end() const {return const_iterator(*this,size());} - //! start const iterator - const_iterator cbegin() const {return const_iterator(*this,0);} - //! end const iterator - const_iterator cend() const {return const_iterator(*this,size());} - //! reverse start iterator - reverse_iterator rbegin() {return reverse_iterator(end());} - //! reverse end iterator - reverse_iterator rend() {return reverse_iterator(begin());} - //! reverse start const iterator - const_reverse_iterator rbegin() const {return const_reverse_iterator(end());} - //! reverse end const iterator - const_reverse_iterator rend() const {return const_reverse_iterator(begin());} - //! reverse start const iterator - const_reverse_iterator crbegin() const {return const_reverse_iterator(end());} - //! reverse end const iterator - const_reverse_iterator crend() const {return const_reverse_iterator(begin());} - //! the first item - reference front() { - __TBB_ASSERT( size()>0, NULL); - const segment_value_t& segment_value = my_segment[0].template load(); - return (segment_value.template pointer())[0]; - } - //! the first item const - const_reference front() const { - __TBB_ASSERT( size()>0, NULL); - const segment_value_t& segment_value = my_segment[0].template load(); - return (segment_value.template pointer())[0]; - } - //! the last item - reference back() { - __TBB_ASSERT( size()>0, NULL); - return internal_subscript( size()-1 ); - } - //! the last item const - const_reference back() const { - __TBB_ASSERT( size()>0, NULL); - return internal_subscript( size()-1 ); - } - //! return allocator object - allocator_type get_allocator() const { return this->my_allocator; } - - //! assign n items by copying t item - void assign(size_type n, const_reference t) { - clear(); - internal_resize( n, sizeof(T), max_size(), static_cast(&t), &destroy_array, &initialize_array_by ); - } - - //! assign range [first, last) - template - void assign(I first, I last) { - clear(); internal_assign_range( first, last, static_cast::is_integer> *>(0) ); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! assigns an initializer list - void assign(std::initializer_list init_list) { - clear(); internal_assign_iterators( init_list.begin(), init_list.end()); - } -#endif //# __TBB_INITIALIZER_LISTS_PRESENT - - //! swap two instances - void swap(concurrent_vector &vector) { - typedef typename tbb::internal::allocator_traits::propagate_on_container_swap pocs_t; - if( this != &vector && (this->my_allocator == vector.my_allocator || pocs_t::value) ) { - concurrent_vector_base_v3::internal_swap(static_cast(vector)); - tbb::internal::allocator_swap(this->my_allocator, vector.my_allocator, pocs_t()); - } - } - - //! Clear container while keeping memory allocated. - /** To free up the memory, use in conjunction with method compact(). Not thread safe **/ - void clear() { - internal_clear(&destroy_array); - } - - //! Clear and destroy vector. - ~concurrent_vector() { - segment_t *table = my_segment.load(); - internal_free_segments( table, internal_clear(&destroy_array), my_first_block.load() ); - // base class destructor call should be then - } - - const internal::concurrent_vector_base_v3 &internal_vector_base() const { return *this; } -private: - //! Allocate k items - static void *internal_allocator(internal::concurrent_vector_base_v3 &vb, size_t k) { - return static_cast&>(vb).my_allocator.allocate(k); - } - //! Free k segments from table - void internal_free_segments(segment_t table[], segment_index_t k, segment_index_t first_block); - - //! Get reference to element at given index. - T& internal_subscript( size_type index ) const; - - //! Get reference to element at given index with errors checks - T& internal_subscript_with_exceptions( size_type index ) const; - - //! assign n items by copying t - void internal_assign_n(size_type n, const_pointer p) { - internal_resize( n, sizeof(T), max_size(), static_cast(p), &destroy_array, p? &initialize_array_by : &initialize_array ); - } - - //! True/false function override helper - /* Functions declarations: - * void foo(is_integer_tag*); - * void foo(is_integer_tag*); - * Usage example: - * foo(static_cast::is_integer>*>(0)); - */ - template class is_integer_tag; - - //! assign integer items by copying when arguments are treated as iterators. See C++ Standard 2003 23.1.1p9 - template - void internal_assign_range(I first, I last, is_integer_tag *) { - internal_assign_n(static_cast(first), &static_cast(last)); - } - //! inline proxy assign by iterators - template - void internal_assign_range(I first, I last, is_integer_tag *) { - internal_assign_iterators(first, last); - } - //! assign by iterators - template - void internal_assign_iterators(I first, I last); - - //these functions are marked __TBB_EXPORTED_FUNC as they are called from within the library - - //! Construct n instances of T, starting at "begin". - static void __TBB_EXPORTED_FUNC initialize_array( void* begin, const void*, size_type n ); - - //! Copy-construct n instances of T, starting at "begin". - static void __TBB_EXPORTED_FUNC initialize_array_by( void* begin, const void* src, size_type n ); - - //! Copy-construct n instances of T by copying single element pointed to by src, starting at "dst". - static void __TBB_EXPORTED_FUNC copy_array( void* dst, const void* src, size_type n ); - -#if __TBB_MOVE_IF_NOEXCEPT_PRESENT - //! Either opy or move-construct n instances of T, starting at "dst" by copying according element of src array. - static void __TBB_EXPORTED_FUNC move_array_if_noexcept( void* dst, const void* src, size_type n ); -#endif //__TBB_MOVE_IF_NO_EXCEPT_PRESENT - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! Move-construct n instances of T, starting at "dst" by copying according element of src array. - static void __TBB_EXPORTED_FUNC move_array( void* dst, const void* src, size_type n ); - - //! Move-assign (using operator=) n instances of T, starting at "dst" by assigning according element of src array. - static void __TBB_EXPORTED_FUNC move_assign_array( void* dst, const void* src, size_type n ); -#endif - //! Copy-construct n instances of T, starting at "dst" by iterator range of [p_type_erased_iterator, p_type_erased_iterator+n). - template - static void __TBB_EXPORTED_FUNC copy_range( void* dst, const void* p_type_erased_iterator, size_type n ); - - //! Assign (using operator=) n instances of T, starting at "dst" by assigning according element of src array. - static void __TBB_EXPORTED_FUNC assign_array( void* dst, const void* src, size_type n ); - - //! Destroy n instances of T, starting at "begin". - static void __TBB_EXPORTED_FUNC destroy_array( void* begin, size_type n ); - - //! Exception-aware helper class for filling a segment by exception-danger operators of user class - class internal_loop_guide : internal::no_copy { - public: - const pointer array; - const size_type n; - size_type i; - - static const T* as_const_pointer(const void *ptr) { return static_cast(ptr); } - static T* as_pointer(const void *src) { return static_cast(const_cast(src)); } - - internal_loop_guide(size_type ntrials, void *ptr) - : array(as_pointer(ptr)), n(ntrials), i(0) {} - void init() { for(; i < n; ++i) new( &array[i] ) T(); } - void init(const void *src) { for(; i < n; ++i) new( &array[i] ) T(*as_const_pointer(src)); } - void copy(const void *src) { for(; i < n; ++i) new( &array[i] ) T(as_const_pointer(src)[i]); } - void assign(const void *src) { for(; i < n; ++i) array[i] = as_const_pointer(src)[i]; } -#if __TBB_CPP11_RVALUE_REF_PRESENT - void move_assign(const void *src) { for(; i < n; ++i) array[i] = std::move(as_pointer(src)[i]); } - void move_construct(const void *src) { for(; i < n; ++i) new( &array[i] ) T( std::move(as_pointer(src)[i]) ); } -#endif -#if __TBB_MOVE_IF_NOEXCEPT_PRESENT - void move_construct_if_noexcept(const void *src) { for(; i < n; ++i) new( &array[i] ) T( std::move_if_noexcept(as_pointer(src)[i]) ); } -#endif //__TBB_MOVE_IF_NOEXCEPT_PRESENT - - //TODO: rename to construct_range - template void iterate(I &src) { for(; i < n; ++i, ++src) new( &array[i] ) T( *src ); } - ~internal_loop_guide() { - if(i < n) {// if an exception was raised, fill the rest of items with zeros - internal::handle_unconstructed_elements(array+i, n-i); - } - } - }; - - struct push_back_helper : internal::no_copy{ - struct element_construction_guard : internal::no_copy{ - pointer element; - - element_construction_guard(pointer an_element) : element (an_element){} - void dismiss(){ element = NULL; } - ~element_construction_guard(){ - if (element){ - internal::handle_unconstructed_elements(element, 1); - } - } - }; - - concurrent_vector & v; - size_type k; - element_construction_guard g; - - push_back_helper(concurrent_vector & vector) : - v(vector), - g (static_cast(v.internal_push_back(sizeof(T),k))) - {} - - pointer internal_push_back_result(){ return g.element;} - iterator return_iterator_and_dismiss(){ - pointer ptr = g.element; - g.dismiss(); - return iterator(v, k, ptr); - } - }; -}; - -#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT -// Deduction guide for the constructor from two iterators -template::value_type, - typename A = cache_aligned_allocator -> concurrent_vector(I, I, const A& = A()) --> concurrent_vector; - -// Deduction guide for the constructor from a vector and allocator -template -concurrent_vector(const concurrent_vector &, const A2 &) --> concurrent_vector; - -// Deduction guide for the constructor from an initializer_list -template -> concurrent_vector(std::initializer_list, const A& = A()) --> concurrent_vector; -#endif /* __TBB_CPP17_DEDUCTION_GUIDES_PRESENT */ - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) -#pragma warning (push) -#pragma warning (disable: 4701) // potentially uninitialized local variable "old" -#endif -template -void concurrent_vector::shrink_to_fit() { - internal_segments_table old; - __TBB_TRY { - internal_array_op2 copy_or_move_array = -#if __TBB_MOVE_IF_NOEXCEPT_PRESENT - &move_array_if_noexcept -#else - ©_array -#endif - ; - if( internal_compact( sizeof(T), &old, &destroy_array, copy_or_move_array ) ) - internal_free_segments( old.table, pointers_per_long_table, old.first_block ); // free joined and unnecessary segments - } __TBB_CATCH(...) { - if( old.first_block ) // free segment allocated for compacting. Only for support of exceptions in ctor of user T[ype] - internal_free_segments( old.table, 1, old.first_block ); - __TBB_RETHROW(); - } -} -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) -#pragma warning (pop) -#endif // warning 4701 is back - -template -void concurrent_vector::internal_free_segments(segment_t table[], segment_index_t k, segment_index_t first_block) { - // Free the arrays - while( k > first_block ) { - --k; - segment_value_t segment_value = table[k].load(); - table[k].store(segment_not_used()); - if( segment_value == segment_allocated() ) // check for correct segment pointer - this->my_allocator.deallocate( (segment_value.pointer()), segment_size(k) ); - } - segment_value_t segment_value = table[0].load(); - if( segment_value == segment_allocated() ) { - __TBB_ASSERT( first_block > 0, NULL ); - while(k > 0) table[--k].store(segment_not_used()); - this->my_allocator.deallocate( (segment_value.pointer()), segment_size(first_block) ); - } -} - -template -T& concurrent_vector::internal_subscript( size_type index ) const { - //TODO: unify both versions of internal_subscript - __TBB_ASSERT( index < my_early_size, "index out of bounds" ); - size_type j = index; - segment_index_t k = segment_base_index_of( j ); - __TBB_ASSERT( my_segment.load() != my_storage || k < pointers_per_short_table, "index is being allocated" ); - //no need in load with acquire (load) since thread works in own space or gets - //the information about added elements via some form of external synchronization - //TODO: why not make a load of my_segment relaxed as well ? - //TODO: add an assertion that my_segment[k] is properly aligned to please ITT - segment_value_t segment_value = my_segment[k].template load(); - __TBB_ASSERT( segment_value != segment_allocation_failed(), "the instance is broken by bad allocation. Use at() instead" ); - __TBB_ASSERT( segment_value != segment_not_used(), "index is being allocated" ); - return (( segment_value.pointer()))[j]; -} - -template -T& concurrent_vector::internal_subscript_with_exceptions( size_type index ) const { - if( index >= my_early_size ) - internal::throw_exception(internal::eid_out_of_range); // throw std::out_of_range - size_type j = index; - segment_index_t k = segment_base_index_of( j ); - //TODO: refactor this condition into separate helper function, e.g. fits_into_small_table - if( my_segment.load() == my_storage && k >= pointers_per_short_table ) - internal::throw_exception(internal::eid_segment_range_error); // throw std::range_error - // no need in load with acquire (load) since thread works in own space or gets - //the information about added elements via some form of external synchronization - //TODO: why not make a load of my_segment relaxed as well ? - //TODO: add an assertion that my_segment[k] is properly aligned to please ITT - segment_value_t segment_value = my_segment[k].template load(); - enforce_segment_allocated(segment_value, internal::eid_index_range_error); - return (segment_value.pointer())[j]; -} - -template template -void concurrent_vector::internal_assign_iterators(I first, I last) { - __TBB_ASSERT(my_early_size == 0, NULL); - size_type n = std::distance(first, last); - if( !n ) return; - internal_reserve(n, sizeof(T), max_size()); - my_early_size = n; - segment_index_t k = 0; - //TODO: unify segment iteration code with concurrent_base_v3::helper - size_type sz = segment_size( my_first_block ); - while( sz < n ) { - internal_loop_guide loop(sz, my_segment[k].template load().template pointer()); - loop.iterate(first); - n -= sz; - if( !k ) k = my_first_block; - else { ++k; sz <<= 1; } - } - internal_loop_guide loop(n, my_segment[k].template load().template pointer()); - loop.iterate(first); -} - -template -void concurrent_vector::initialize_array( void* begin, const void *, size_type n ) { - internal_loop_guide loop(n, begin); loop.init(); -} - -template -void concurrent_vector::initialize_array_by( void* begin, const void *src, size_type n ) { - internal_loop_guide loop(n, begin); loop.init(src); -} - -template -void concurrent_vector::copy_array( void* dst, const void* src, size_type n ) { - internal_loop_guide loop(n, dst); loop.copy(src); -} - -#if __TBB_CPP11_RVALUE_REF_PRESENT -template -void concurrent_vector::move_array( void* dst, const void* src, size_type n ) { - internal_loop_guide loop(n, dst); loop.move_construct(src); -} -template -void concurrent_vector::move_assign_array( void* dst, const void* src, size_type n ) { - internal_loop_guide loop(n, dst); loop.move_assign(src); -} -#endif - -#if __TBB_MOVE_IF_NOEXCEPT_PRESENT -template -void concurrent_vector::move_array_if_noexcept( void* dst, const void* src, size_type n ) { - internal_loop_guide loop(n, dst); loop.move_construct_if_noexcept(src); -} -#endif //__TBB_MOVE_IF_NOEXCEPT_PRESENT - -template -template -void concurrent_vector::copy_range( void* dst, const void* p_type_erased_iterator, size_type n ){ - internal_loop_guide loop(n, dst); - loop.iterate( *(static_cast(const_cast(p_type_erased_iterator))) ); -} - -template -void concurrent_vector::assign_array( void* dst, const void* src, size_type n ) { - internal_loop_guide loop(n, dst); loop.assign(src); -} - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - // Workaround for overzealous compiler warning - #pragma warning (push) - #pragma warning (disable: 4189) -#endif -template -void concurrent_vector::destroy_array( void* begin, size_type n ) { - T* array = static_cast(begin); - for( size_type j=n; j>0; --j ) - array[j-1].~T(); // destructors are supposed to not throw any exceptions -} -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warning 4189 is back - -// concurrent_vector's template functions -template -inline bool operator==(const concurrent_vector &a, const concurrent_vector &b) { - //TODO: call size() only once per vector (in operator==) - // Simply: return a.size() == b.size() && std::equal(a.begin(), a.end(), b.begin()); - if(a.size() != b.size()) return false; - typename concurrent_vector::const_iterator i(a.begin()); - typename concurrent_vector::const_iterator j(b.begin()); - for(; i != a.end(); ++i, ++j) - if( !(*i == *j) ) return false; - return true; -} - -template -inline bool operator!=(const concurrent_vector &a, const concurrent_vector &b) -{ return !(a == b); } - -template -inline bool operator<(const concurrent_vector &a, const concurrent_vector &b) -{ return (std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end())); } - -template -inline bool operator>(const concurrent_vector &a, const concurrent_vector &b) -{ return b < a; } - -template -inline bool operator<=(const concurrent_vector &a, const concurrent_vector &b) -{ return !(b < a); } - -template -inline bool operator>=(const concurrent_vector &a, const concurrent_vector &b) -{ return !(a < b); } - -template -inline void swap(concurrent_vector &a, concurrent_vector &b) -{ a.swap( b ); } - -} // namespace tbb - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warning 4267,4127 are back - - -#undef __TBB_concurrent_vector_H_include_area -#include "internal/_warning_suppress_disable_notice.h" - -#endif /* __TBB_concurrent_vector_H */ +#include "../oneapi/tbb/concurrent_vector.h" diff --git a/src/3rd party/tbb/critical_section.h b/src/3rd party/tbb/critical_section.h deleted file mode 100644 index 13371797ed..0000000000 --- a/src/3rd party/tbb/critical_section.h +++ /dev/null @@ -1,147 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include "internal/_deprecated_header_message_guard.h" - -#if !defined(__TBB_show_deprecation_message_critical_section_H) && defined(__TBB_show_deprecated_header_message) -#define __TBB_show_deprecation_message_critical_section_H -#pragma message("TBB Warning: tbb/critical_section.h is deprecated. For details, please see Deprecated Features appendix in the TBB reference manual.") -#endif - -#if defined(__TBB_show_deprecated_header_message) -#undef __TBB_show_deprecated_header_message -#endif - -#ifndef _TBB_CRITICAL_SECTION_H_ -#define _TBB_CRITICAL_SECTION_H_ - -#define __TBB_critical_section_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#else -#include -#include -#endif // _WIN32||WIN64 - -#include "tbb_stddef.h" -#include "tbb_thread.h" -#include "tbb_exception.h" - -#include "tbb_profiling.h" - -namespace tbb { - - namespace internal { -class critical_section_v4 : internal::no_copy { -#if _WIN32||_WIN64 - CRITICAL_SECTION my_impl; -#else - pthread_mutex_t my_impl; -#endif - tbb_thread::id my_tid; -public: - - void __TBB_EXPORTED_METHOD internal_construct(); - - critical_section_v4() { -#if _WIN32||_WIN64 - InitializeCriticalSectionEx( &my_impl, 4000, 0 ); -#else - pthread_mutex_init(&my_impl, NULL); -#endif - internal_construct(); - } - - ~critical_section_v4() { - __TBB_ASSERT(my_tid == tbb_thread::id(), "Destroying a still-held critical section"); -#if _WIN32||_WIN64 - DeleteCriticalSection(&my_impl); -#else - pthread_mutex_destroy(&my_impl); -#endif - } - - class scoped_lock : internal::no_copy { - private: - critical_section_v4 &my_crit; - public: - scoped_lock( critical_section_v4& lock_me) :my_crit(lock_me) { - my_crit.lock(); - } - - ~scoped_lock() { - my_crit.unlock(); - } - }; - - void lock() { - tbb_thread::id local_tid = this_tbb_thread::get_id(); - if(local_tid == my_tid) throw_exception( eid_improper_lock ); -#if _WIN32||_WIN64 - EnterCriticalSection( &my_impl ); -#else - int rval = pthread_mutex_lock(&my_impl); - __TBB_ASSERT_EX(!rval, "critical_section::lock: pthread_mutex_lock failed"); -#endif - __TBB_ASSERT(my_tid == tbb_thread::id(), NULL); - my_tid = local_tid; - } - - bool try_lock() { - bool gotlock; - tbb_thread::id local_tid = this_tbb_thread::get_id(); - if(local_tid == my_tid) return false; -#if _WIN32||_WIN64 - gotlock = TryEnterCriticalSection( &my_impl ) != 0; -#else - int rval = pthread_mutex_trylock(&my_impl); - // valid returns are 0 (locked) and [EBUSY] - __TBB_ASSERT(rval == 0 || rval == EBUSY, "critical_section::trylock: pthread_mutex_trylock failed"); - gotlock = rval == 0; -#endif - if(gotlock) { - my_tid = local_tid; - } - return gotlock; - } - - void unlock() { - __TBB_ASSERT(this_tbb_thread::get_id() == my_tid, "thread unlocking critical_section is not thread that locked it"); - my_tid = tbb_thread::id(); -#if _WIN32||_WIN64 - LeaveCriticalSection( &my_impl ); -#else - int rval = pthread_mutex_unlock(&my_impl); - __TBB_ASSERT_EX(!rval, "critical_section::unlock: pthread_mutex_unlock failed"); -#endif - } - - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = true; -}; // critical_section_v4 -} // namespace internal -__TBB_DEPRECATED_VERBOSE_MSG("tbb::critical_section is deprecated, use std::mutex") typedef internal::critical_section_v4 critical_section; - -__TBB_DEFINE_PROFILING_SET_NAME(critical_section) -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_critical_section_H_include_area - -#endif // _TBB_CRITICAL_SECTION_H_ diff --git a/src/3rd party/tbb/enumerable_thread_specific.h b/src/3rd party/tbb/enumerable_thread_specific.h index 2af1f14d2d..9d6050d64f 100644 --- a/src/3rd party/tbb/enumerable_thread_specific.h +++ b/src/3rd party/tbb/enumerable_thread_specific.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,1160 +14,4 @@ limitations under the License. */ -#ifndef __TBB_enumerable_thread_specific_H -#define __TBB_enumerable_thread_specific_H - -#define __TBB_enumerable_thread_specific_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "atomic.h" -#include "concurrent_vector.h" -#include "tbb_thread.h" -#include "tbb_allocator.h" -#include "cache_aligned_allocator.h" -#include "aligned_space.h" -#include "internal/_template_helpers.h" -#include "internal/_tbb_hash_compare_impl.h" -#include "tbb_profiling.h" -#include // for memcpy - -#if __TBB_PREVIEW_RESUMABLE_TASKS -#include "task.h" // for task::suspend_point -#endif - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#else -#include -#endif - -#define __TBB_ETS_USE_CPP11 \ - (__TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT \ - && __TBB_CPP11_DECLTYPE_PRESENT && __TBB_CPP11_LAMBDAS_PRESENT) - -namespace tbb { - -//! enum for selecting between single key and key-per-instance versions -enum ets_key_usage_type { - ets_key_per_instance - , ets_no_key -#if __TBB_PREVIEW_RESUMABLE_TASKS - , ets_suspend_aware -#endif -}; - -namespace interface6 { - - // Forward declaration to use in internal classes - template - class enumerable_thread_specific; - - //! @cond - namespace internal { - - using namespace tbb::internal; - - template - struct ets_key_selector { - typedef tbb_thread::id key_type; - static key_type current_key() { - return tbb::internal::thread_get_id_v3(); - } - }; - -#if __TBB_PREVIEW_RESUMABLE_TASKS - template <> - struct ets_key_selector { - typedef task::suspend_point key_type; - static key_type current_key() { - return internal_current_suspend_point(); - } - }; - - inline task::suspend_point atomic_compare_and_swap(task::suspend_point& location, - const task::suspend_point& value, const task::suspend_point& comparand) { - return as_atomic(location).compare_and_swap(value, comparand); - } -#endif - - template - class ets_base: tbb::internal::no_copy { - protected: - typedef typename ets_key_selector::key_type key_type; -#if __TBB_PROTECTED_NESTED_CLASS_BROKEN - public: -#endif - struct slot; - - struct array { - array* next; - size_t lg_size; - slot& at( size_t k ) { - return ((slot*)(void*)(this+1))[k]; - } - size_t size() const {return size_t(1)<>(8*sizeof(size_t)-lg_size); - } - }; - struct slot { - key_type key; - void* ptr; - bool empty() const {return key == key_type();} - bool match( key_type k ) const {return key == k;} - bool claim( key_type k ) { - // TODO: maybe claim ptr, because key_type is not guaranteed to fit into word size - return atomic_compare_and_swap(key, k, key_type()) == key_type(); - } - }; -#if __TBB_PROTECTED_NESTED_CLASS_BROKEN - protected: -#endif - - //! Root of linked list of arrays of decreasing size. - /** NULL if and only if my_count==0. - Each array in the list is half the size of its predecessor. */ - atomic my_root; - atomic my_count; - virtual void* create_local() = 0; - virtual void* create_array(size_t _size) = 0; // _size in bytes - virtual void free_array(void* ptr, size_t _size) = 0; // _size in bytes - array* allocate( size_t lg_size ) { - size_t n = size_t(1)<(create_array( sizeof(array)+n*sizeof(slot) )); - a->lg_size = lg_size; - std::memset( a+1, 0, n*sizeof(slot) ); - return a; - } - void free(array* a) { - size_t n = size_t(1)<<(a->lg_size); - free_array( (void *)a, size_t(sizeof(array)+n*sizeof(slot)) ); - } - - ets_base() {my_root=NULL; my_count=0;} - virtual ~ets_base(); // g++ complains if this is not virtual - void* table_lookup( bool& exists ); - void table_clear(); - // The following functions are not used in concurrent context, - // so we don't need synchronization and ITT annotations there. - template - void table_elementwise_copy( const ets_base& other, - void*(*add_element)(ets_base&, void*) ) { - __TBB_ASSERT(!my_root,NULL); - __TBB_ASSERT(!my_count,NULL); - if( !other.my_root ) return; - array* root = my_root = allocate(other.my_root->lg_size); - root->next = NULL; - my_count = other.my_count; - size_t mask = root->mask(); - for( array* r=other.my_root; r; r=r->next ) { - for( size_t i=0; isize(); ++i ) { - slot& s1 = r->at(i); - if( !s1.empty() ) { - for( size_t j = root->start(tbb::tbb_hash()(s1.key)); ; j=(j+1)&mask ) { - slot& s2 = root->at(j); - if( s2.empty() ) { - s2.ptr = add_element(static_cast&>(*this), s1.ptr); - s2.key = s1.key; - break; - } - else if( s2.match(s1.key) ) - break; - } - } - } - } - } - void table_swap( ets_base& other ) { - __TBB_ASSERT(this!=&other, "Don't swap an instance with itself"); - tbb::internal::swap(my_root, other.my_root); - tbb::internal::swap(my_count, other.my_count); - } - }; - - template - ets_base::~ets_base() { - __TBB_ASSERT(!my_root, NULL); - } - - template - void ets_base::table_clear() { - while( array* r = my_root ) { - my_root = r->next; - free(r); - } - my_count = 0; - } - - template - void* ets_base::table_lookup( bool& exists ) { - const key_type k = ets_key_selector::current_key(); - - __TBB_ASSERT(k != key_type(),NULL); - void* found; - size_t h = tbb::tbb_hash()(k); - for( array* r=my_root; r; r=r->next ) { - call_itt_notify(acquired,r); - size_t mask=r->mask(); - for(size_t i = r->start(h); ;i=(i+1)&mask) { - slot& s = r->at(i); - if( s.empty() ) break; - if( s.match(k) ) { - if( r==my_root ) { - // Success at top level - exists = true; - return s.ptr; - } else { - // Success at some other level. Need to insert at top level. - exists = true; - found = s.ptr; - goto insert; - } - } - } - } - // Key does not yet exist. The density of slots in the table does not exceed 0.5, - // for if this will occur a new table is allocated with double the current table - // size, which is swapped in as the new root table. So an empty slot is guaranteed. - exists = false; - found = create_local(); - { - size_t c = ++my_count; - array* r = my_root; - call_itt_notify(acquired,r); - if( !r || c>r->size()/2 ) { - size_t s = r ? r->lg_size : 2; - while( c>size_t(1)<<(s-1) ) ++s; - array* a = allocate(s); - for(;;) { - a->next = r; - call_itt_notify(releasing,a); - array* new_r = my_root.compare_and_swap(a,r); - if( new_r==r ) break; - call_itt_notify(acquired, new_r); - if( new_r->lg_size>=s ) { - // Another thread inserted an equal or bigger array, so our array is superfluous. - free(a); - break; - } - r = new_r; - } - } - } - insert: - // Whether a slot has been found in an older table, or if it has been inserted at this level, - // it has already been accounted for in the total. Guaranteed to be room for it, and it is - // not present, so search for empty slot and use it. - array* ir = my_root; - call_itt_notify(acquired, ir); - size_t mask = ir->mask(); - for(size_t i = ir->start(h);;i=(i+1)&mask) { - slot& s = ir->at(i); - if( s.empty() ) { - if( s.claim(k) ) { - s.ptr = found; - return found; - } - } - } - } - - //! Specialization that exploits native TLS - template <> - class ets_base: public ets_base { - typedef ets_base super; -#if _WIN32||_WIN64 -#if __TBB_WIN8UI_SUPPORT - typedef DWORD tls_key_t; - void create_key() { my_key = FlsAlloc(NULL); } - void destroy_key() { FlsFree(my_key); } - void set_tls(void * value) { FlsSetValue(my_key, (LPVOID)value); } - void* get_tls() { return (void *)FlsGetValue(my_key); } -#else - typedef DWORD tls_key_t; - void create_key() { my_key = TlsAlloc(); } - void destroy_key() { TlsFree(my_key); } - void set_tls(void * value) { TlsSetValue(my_key, (LPVOID)value); } - void* get_tls() { return (void *)TlsGetValue(my_key); } -#endif -#else - typedef pthread_key_t tls_key_t; - void create_key() { pthread_key_create(&my_key, NULL); } - void destroy_key() { pthread_key_delete(my_key); } - void set_tls( void * value ) const { pthread_setspecific(my_key, value); } - void* get_tls() const { return pthread_getspecific(my_key); } -#endif - tls_key_t my_key; - virtual void* create_local() __TBB_override = 0; - virtual void* create_array(size_t _size) __TBB_override = 0; // _size in bytes - virtual void free_array(void* ptr, size_t _size) __TBB_override = 0; // size in bytes - protected: - ets_base() {create_key();} - ~ets_base() {destroy_key();} - void* table_lookup( bool& exists ) { - void* found = get_tls(); - if( found ) { - exists=true; - } else { - found = super::table_lookup(exists); - set_tls(found); - } - return found; - } - void table_clear() { - destroy_key(); - create_key(); - super::table_clear(); - } - void table_swap( ets_base& other ) { - using std::swap; - __TBB_ASSERT(this!=&other, "Don't swap an instance with itself"); - swap(my_key, other.my_key); - super::table_swap(other); - } - }; - - //! Random access iterator for traversing the thread local copies. - template< typename Container, typename Value > - class enumerable_thread_specific_iterator -#if defined(_WIN64) && defined(_MSC_VER) - // Ensure that Microsoft's internal template function _Val_type works correctly. - : public std::iterator -#endif /* defined(_WIN64) && defined(_MSC_VER) */ - { - //! current position in the concurrent_vector - - Container *my_container; - typename Container::size_type my_index; - mutable Value *my_value; - - template - friend enumerable_thread_specific_iterator - operator+( ptrdiff_t offset, const enumerable_thread_specific_iterator& v ); - - template - friend bool operator==( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ); - - template - friend bool operator<( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ); - - template - friend ptrdiff_t operator-( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ); - - template - friend class enumerable_thread_specific_iterator; - - public: - - enumerable_thread_specific_iterator( const Container &container, typename Container::size_type index ) : - my_container(&const_cast(container)), my_index(index), my_value(NULL) {} - - //! Default constructor - enumerable_thread_specific_iterator() : my_container(NULL), my_index(0), my_value(NULL) {} - - template - enumerable_thread_specific_iterator( const enumerable_thread_specific_iterator& other ) : - my_container( other.my_container ), my_index( other.my_index), my_value( const_cast(other.my_value) ) {} - - enumerable_thread_specific_iterator operator+( ptrdiff_t offset ) const { - return enumerable_thread_specific_iterator(*my_container, my_index + offset); - } - - enumerable_thread_specific_iterator &operator+=( ptrdiff_t offset ) { - my_index += offset; - my_value = NULL; - return *this; - } - - enumerable_thread_specific_iterator operator-( ptrdiff_t offset ) const { - return enumerable_thread_specific_iterator( *my_container, my_index-offset ); - } - - enumerable_thread_specific_iterator &operator-=( ptrdiff_t offset ) { - my_index -= offset; - my_value = NULL; - return *this; - } - - Value& operator*() const { - Value* value = my_value; - if( !value ) { - value = my_value = (*my_container)[my_index].value(); - } - __TBB_ASSERT( value==(*my_container)[my_index].value(), "corrupt cache" ); - return *value; - } - - Value& operator[]( ptrdiff_t k ) const { - return (*my_container)[my_index + k].value; - } - - Value* operator->() const {return &operator*();} - - enumerable_thread_specific_iterator& operator++() { - ++my_index; - my_value = NULL; - return *this; - } - - enumerable_thread_specific_iterator& operator--() { - --my_index; - my_value = NULL; - return *this; - } - - //! Post increment - enumerable_thread_specific_iterator operator++(int) { - enumerable_thread_specific_iterator result = *this; - ++my_index; - my_value = NULL; - return result; - } - - //! Post decrement - enumerable_thread_specific_iterator operator--(int) { - enumerable_thread_specific_iterator result = *this; - --my_index; - my_value = NULL; - return result; - } - - // STL support - typedef ptrdiff_t difference_type; - typedef Value value_type; - typedef Value* pointer; - typedef Value& reference; - typedef std::random_access_iterator_tag iterator_category; - }; - - template - enumerable_thread_specific_iterator - operator+( ptrdiff_t offset, const enumerable_thread_specific_iterator& v ) { - return enumerable_thread_specific_iterator( v.my_container, v.my_index + offset ); - } - - template - bool operator==( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return i.my_index==j.my_index && i.my_container == j.my_container; - } - - template - bool operator!=( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return !(i==j); - } - - template - bool operator<( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return i.my_index - bool operator>( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return j - bool operator>=( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return !(i - bool operator<=( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return !(j - ptrdiff_t operator-( const enumerable_thread_specific_iterator& i, - const enumerable_thread_specific_iterator& j ) { - return i.my_index-j.my_index; - } - - template - class segmented_iterator -#if defined(_WIN64) && defined(_MSC_VER) - : public std::iterator -#endif - { - template - friend bool operator==(const segmented_iterator& i, const segmented_iterator& j); - - template - friend bool operator!=(const segmented_iterator& i, const segmented_iterator& j); - - template - friend class segmented_iterator; - - public: - - segmented_iterator() {my_segcont = NULL;} - - segmented_iterator( const SegmentedContainer& _segmented_container ) : - my_segcont(const_cast(&_segmented_container)), - outer_iter(my_segcont->end()) { } - - ~segmented_iterator() {} - - typedef typename SegmentedContainer::iterator outer_iterator; - typedef typename SegmentedContainer::value_type InnerContainer; - typedef typename InnerContainer::iterator inner_iterator; - - // STL support - typedef ptrdiff_t difference_type; - typedef Value value_type; - typedef typename SegmentedContainer::size_type size_type; - typedef Value* pointer; - typedef Value& reference; - typedef std::input_iterator_tag iterator_category; - - // Copy Constructor - template - segmented_iterator(const segmented_iterator& other) : - my_segcont(other.my_segcont), - outer_iter(other.outer_iter), - // can we assign a default-constructed iterator to inner if we're at the end? - inner_iter(other.inner_iter) - {} - - // assignment - template - segmented_iterator& operator=( const segmented_iterator& other) { - if(this != &other) { - my_segcont = other.my_segcont; - outer_iter = other.outer_iter; - if(outer_iter != my_segcont->end()) inner_iter = other.inner_iter; - } - return *this; - } - - // allow assignment of outer iterator to segmented iterator. Once it is - // assigned, move forward until a non-empty inner container is found or - // the end of the outer container is reached. - segmented_iterator& operator=(const outer_iterator& new_outer_iter) { - __TBB_ASSERT(my_segcont != NULL, NULL); - // check that this iterator points to something inside the segmented container - for(outer_iter = new_outer_iter ;outer_iter!=my_segcont->end(); ++outer_iter) { - if( !outer_iter->empty() ) { - inner_iter = outer_iter->begin(); - break; - } - } - return *this; - } - - // pre-increment - segmented_iterator& operator++() { - advance_me(); - return *this; - } - - // post-increment - segmented_iterator operator++(int) { - segmented_iterator tmp = *this; - operator++(); - return tmp; - } - - bool operator==(const outer_iterator& other_outer) const { - __TBB_ASSERT(my_segcont != NULL, NULL); - return (outer_iter == other_outer && - (outer_iter == my_segcont->end() || inner_iter == outer_iter->begin())); - } - - bool operator!=(const outer_iterator& other_outer) const { - return !operator==(other_outer); - - } - - // (i)* RHS - reference operator*() const { - __TBB_ASSERT(my_segcont != NULL, NULL); - __TBB_ASSERT(outer_iter != my_segcont->end(), "Dereferencing a pointer at end of container"); - __TBB_ASSERT(inner_iter != outer_iter->end(), NULL); // should never happen - return *inner_iter; - } - - // i-> - pointer operator->() const { return &operator*();} - - private: - SegmentedContainer* my_segcont; - outer_iterator outer_iter; - inner_iterator inner_iter; - - void advance_me() { - __TBB_ASSERT(my_segcont != NULL, NULL); - __TBB_ASSERT(outer_iter != my_segcont->end(), NULL); // not true if there are no inner containers - __TBB_ASSERT(inner_iter != outer_iter->end(), NULL); // not true if the inner containers are all empty. - ++inner_iter; - while(inner_iter == outer_iter->end() && ++outer_iter != my_segcont->end()) { - inner_iter = outer_iter->begin(); - } - } - }; // segmented_iterator - - template - bool operator==( const segmented_iterator& i, - const segmented_iterator& j ) { - if(i.my_segcont != j.my_segcont) return false; - if(i.my_segcont == NULL) return true; - if(i.outer_iter != j.outer_iter) return false; - if(i.outer_iter == i.my_segcont->end()) return true; - return i.inner_iter == j.inner_iter; - } - - // != - template - bool operator!=( const segmented_iterator& i, - const segmented_iterator& j ) { - return !(i==j); - } - - template - struct construct_by_default: tbb::internal::no_assign { - void construct(void*where) {new(where) T();} // C++ note: the () in T() ensure zero initialization. - construct_by_default( int ) {} - }; - - template - struct construct_by_exemplar: tbb::internal::no_assign { - const T exemplar; - void construct(void*where) {new(where) T(exemplar);} - construct_by_exemplar( const T& t ) : exemplar(t) {} -#if __TBB_ETS_USE_CPP11 - construct_by_exemplar( T&& t ) : exemplar(std::move(t)) {} -#endif - }; - - template - struct construct_by_finit: tbb::internal::no_assign { - Finit f; - void construct(void* where) {new(where) T(f());} - construct_by_finit( const Finit& f_ ) : f(f_) {} -#if __TBB_ETS_USE_CPP11 - construct_by_finit( Finit&& f_ ) : f(std::move(f_)) {} -#endif - }; - -#if __TBB_ETS_USE_CPP11 - template - struct construct_by_args: tbb::internal::no_assign { - internal::stored_pack pack; - void construct(void* where) { - internal::call( [where](const typename strip

::type&... args ){ - new(where) T(args...); - }, pack ); - } - construct_by_args( P&& ... args ) : pack(std::forward

(args)...) {} - }; -#endif - - // storage for initialization function pointer - // TODO: consider removing the template parameter T here and in callback_leaf - template - class callback_base { - public: - // Clone *this - virtual callback_base* clone() const = 0; - // Destruct and free *this - virtual void destroy() = 0; - // Need virtual destructor to satisfy GCC compiler warning - virtual ~callback_base() { } - // Construct T at where - virtual void construct(void* where) = 0; - }; - - template - class callback_leaf: public callback_base, Constructor { -#if __TBB_ETS_USE_CPP11 - template callback_leaf( P&& ... params ) : Constructor(std::forward

(params)...) {} -#else - template callback_leaf( const X& x ) : Constructor(x) {} -#endif - // TODO: make the construction/destruction consistent (use allocator.construct/destroy) - typedef typename tbb::tbb_allocator my_allocator_type; - - callback_base* clone() const __TBB_override { - return make(*this); - } - - void destroy() __TBB_override { - my_allocator_type().destroy(this); - my_allocator_type().deallocate(this,1); - } - - void construct(void* where) __TBB_override { - Constructor::construct(where); - } - public: -#if __TBB_ETS_USE_CPP11 - template - static callback_base* make( P&& ... params ) { - void* where = my_allocator_type().allocate(1); - return new(where) callback_leaf( std::forward

(params)... ); - } -#else - template - static callback_base* make( const X& x ) { - void* where = my_allocator_type().allocate(1); - return new(where) callback_leaf(x); - } -#endif - }; - - //! Template for recording construction of objects in table - /** All maintenance of the space will be done explicitly on push_back, - and all thread local copies must be destroyed before the concurrent - vector is deleted. - - The flag is_built is initialized to false. When the local is - successfully-constructed, set the flag to true or call value_committed(). - If the constructor throws, the flag will be false. - */ - template - struct ets_element { - tbb::aligned_space my_space; - bool is_built; - ets_element() { is_built = false; } // not currently-built - U* value() { return my_space.begin(); } - U* value_committed() { is_built = true; return my_space.begin(); } - ~ets_element() { - if(is_built) { - my_space.begin()->~U(); - is_built = false; - } - } - }; - - // A predicate that can be used for a compile-time compatibility check of ETS instances - // Ideally, it should have been declared inside the ETS class, but unfortunately - // in that case VS2013 does not enable the variadic constructor. - template struct is_compatible_ets { static const bool value = false; }; - template - struct is_compatible_ets< T, enumerable_thread_specific > { static const bool value = internal::is_same_type::value; }; - -#if __TBB_ETS_USE_CPP11 - // A predicate that checks whether, for a variable 'foo' of type T, foo() is a valid expression - template - class is_callable_no_args { - private: - typedef char yes[1]; - typedef char no [2]; - - template static yes& decide( decltype(declval()())* ); - template static no& decide(...); - public: - static const bool value = (sizeof(decide(NULL)) == sizeof(yes)); - }; -#endif - - } // namespace internal - //! @endcond - - //! The enumerable_thread_specific container - /** enumerable_thread_specific has the following properties: - - thread-local copies are lazily created, with default, exemplar or function initialization. - - thread-local copies do not move (during lifetime, and excepting clear()) so the address of a copy is invariant. - - the contained objects need not have operator=() defined if combine is not used. - - enumerable_thread_specific containers may be copy-constructed or assigned. - - thread-local copies can be managed by hash-table, or can be accessed via TLS storage for speed. - - outside of parallel contexts, the contents of all thread-local copies are accessible by iterator or using combine or combine_each methods - - @par Segmented iterator - When the thread-local objects are containers with input_iterators defined, a segmented iterator may - be used to iterate over all the elements of all thread-local copies. - - @par combine and combine_each - - Both methods are defined for enumerable_thread_specific. - - combine() requires the type T have operator=() defined. - - neither method modifies the contents of the object (though there is no guarantee that the applied methods do not modify the object.) - - Both are evaluated in serial context (the methods are assumed to be non-benign.) - - @ingroup containers */ - template , - ets_key_usage_type ETS_key_type=ets_no_key > - class enumerable_thread_specific: internal::ets_base { - - template friend class enumerable_thread_specific; - - typedef internal::padded< internal::ets_element > padded_element; - - //! A generic range, used to create range objects from the iterators - template - class generic_range_type: public blocked_range { - public: - typedef T value_type; - typedef T& reference; - typedef const T& const_reference; - typedef I iterator; - typedef ptrdiff_t difference_type; - generic_range_type( I begin_, I end_, size_t grainsize_ = 1) : blocked_range(begin_,end_,grainsize_) {} - template - generic_range_type( const generic_range_type& r) : blocked_range(r.begin(),r.end(),r.grainsize()) {} - generic_range_type( generic_range_type& r, split ) : blocked_range(r,split()) {} - }; - - typedef typename Allocator::template rebind< padded_element >::other padded_allocator_type; - typedef tbb::concurrent_vector< padded_element, padded_allocator_type > internal_collection_type; - - internal::callback_base *my_construct_callback; - - internal_collection_type my_locals; - - // TODO: consider unifying the callback mechanism for all create_local* methods below - // (likely non-compatible and requires interface version increase) - void* create_local() __TBB_override { - padded_element& lref = *my_locals.grow_by(1); - my_construct_callback->construct(lref.value()); - return lref.value_committed(); - } - - static void* create_local_by_copy( internal::ets_base& base, void* p ) { - enumerable_thread_specific& ets = static_cast(base); - padded_element& lref = *ets.my_locals.grow_by(1); - new(lref.value()) T(*static_cast(p)); - return lref.value_committed(); - } - -#if __TBB_ETS_USE_CPP11 - static void* create_local_by_move( internal::ets_base& base, void* p ) { - enumerable_thread_specific& ets = static_cast(base); - padded_element& lref = *ets.my_locals.grow_by(1); - new(lref.value()) T(std::move(*static_cast(p))); - return lref.value_committed(); - } -#endif - - typedef typename Allocator::template rebind< uintptr_t >::other array_allocator_type; - - // _size is in bytes - void* create_array(size_t _size) __TBB_override { - size_t nelements = (_size + sizeof(uintptr_t) -1) / sizeof(uintptr_t); - return array_allocator_type().allocate(nelements); - } - - void free_array( void* _ptr, size_t _size) __TBB_override { - size_t nelements = (_size + sizeof(uintptr_t) -1) / sizeof(uintptr_t); - array_allocator_type().deallocate( reinterpret_cast(_ptr),nelements); - } - - public: - - //! Basic types - typedef Allocator allocator_type; - typedef T value_type; - typedef T& reference; - typedef const T& const_reference; - typedef T* pointer; - typedef const T* const_pointer; - typedef typename internal_collection_type::size_type size_type; - typedef typename internal_collection_type::difference_type difference_type; - - // Iterator types - typedef typename internal::enumerable_thread_specific_iterator< internal_collection_type, value_type > iterator; - typedef typename internal::enumerable_thread_specific_iterator< internal_collection_type, const value_type > const_iterator; - - // Parallel range types - typedef generic_range_type< iterator > range_type; - typedef generic_range_type< const_iterator > const_range_type; - - //! Default constructor. Each local instance of T is default constructed. - enumerable_thread_specific() : my_construct_callback( - internal::callback_leaf >::make(/*dummy argument*/0) - ){} - - //! Constructor with initializer functor. Each local instance of T is constructed by T(finit()). - template ::type>::value>::type -#endif - > - explicit enumerable_thread_specific( Finit finit ) : my_construct_callback( - internal::callback_leaf >::make( tbb::internal::move(finit) ) - ){} - - //! Constructor with exemplar. Each local instance of T is copy-constructed from the exemplar. - explicit enumerable_thread_specific( const T& exemplar ) : my_construct_callback( - internal::callback_leaf >::make( exemplar ) - ){} - -#if __TBB_ETS_USE_CPP11 - explicit enumerable_thread_specific( T&& exemplar ) : my_construct_callback( - internal::callback_leaf >::make( std::move(exemplar) ) - ){} - - //! Variadic constructor with initializer arguments. Each local instance of T is constructed by T(args...) - template ::type>::value - && !internal::is_compatible_ets::type>::value - && !internal::is_same_type::type>::value - >::type> - enumerable_thread_specific( P1&& arg1, P&& ... args ) : my_construct_callback( - internal::callback_leaf >::make( std::forward(arg1), std::forward

(args)... ) - ){} -#endif - - //! Destructor - ~enumerable_thread_specific() { - if(my_construct_callback) my_construct_callback->destroy(); - // Deallocate the hash table before overridden free_array() becomes inaccessible - this->internal::ets_base::table_clear(); - } - - //! returns reference to local, discarding exists - reference local() { - bool exists; - return local(exists); - } - - //! Returns reference to calling thread's local copy, creating one if necessary - reference local(bool& exists) { - void* ptr = this->table_lookup(exists); - return *(T*)ptr; - } - - //! Get the number of local copies - size_type size() const { return my_locals.size(); } - - //! true if there have been no local copies created - bool empty() const { return my_locals.empty(); } - - //! begin iterator - iterator begin() { return iterator( my_locals, 0 ); } - //! end iterator - iterator end() { return iterator(my_locals, my_locals.size() ); } - - //! begin const iterator - const_iterator begin() const { return const_iterator(my_locals, 0); } - - //! end const iterator - const_iterator end() const { return const_iterator(my_locals, my_locals.size()); } - - //! Get range for parallel algorithms - range_type range( size_t grainsize=1 ) { return range_type( begin(), end(), grainsize ); } - - //! Get const range for parallel algorithms - const_range_type range( size_t grainsize=1 ) const { return const_range_type( begin(), end(), grainsize ); } - - //! Destroys local copies - void clear() { - my_locals.clear(); - this->table_clear(); - // callback is not destroyed - } - - private: - - template - void internal_copy(const enumerable_thread_specific& other) { -#if __TBB_ETS_USE_CPP11 && TBB_USE_ASSERT - // this tests is_compatible_ets - __TBB_STATIC_ASSERT( (internal::is_compatible_ets::type>::value), "is_compatible_ets fails" ); -#endif - // Initialize my_construct_callback first, so that it is valid even if rest of this routine throws an exception. - my_construct_callback = other.my_construct_callback->clone(); - __TBB_ASSERT(my_locals.size()==0,NULL); - my_locals.reserve(other.size()); - this->table_elementwise_copy( other, create_local_by_copy ); - } - - void internal_swap(enumerable_thread_specific& other) { - using std::swap; - __TBB_ASSERT( this!=&other, NULL ); - swap(my_construct_callback, other.my_construct_callback); - // concurrent_vector::swap() preserves storage space, - // so addresses to the vector kept in ETS hash table remain valid. - swap(my_locals, other.my_locals); - this->internal::ets_base::table_swap(other); - } - -#if __TBB_ETS_USE_CPP11 - template - void internal_move(enumerable_thread_specific&& other) { -#if TBB_USE_ASSERT - // this tests is_compatible_ets - __TBB_STATIC_ASSERT( (internal::is_compatible_ets::type>::value), "is_compatible_ets fails" ); -#endif - my_construct_callback = other.my_construct_callback; - other.my_construct_callback = NULL; - __TBB_ASSERT(my_locals.size()==0,NULL); - my_locals.reserve(other.size()); - this->table_elementwise_copy( other, create_local_by_move ); - } -#endif - - public: - - enumerable_thread_specific( const enumerable_thread_specific& other ) - : internal::ets_base() /* prevents GCC warnings with -Wextra */ - { - internal_copy(other); - } - - template - enumerable_thread_specific( const enumerable_thread_specific& other ) - { - internal_copy(other); - } - -#if __TBB_ETS_USE_CPP11 - enumerable_thread_specific( enumerable_thread_specific&& other ) : my_construct_callback() - { - internal_swap(other); - } - - template - enumerable_thread_specific( enumerable_thread_specific&& other ) : my_construct_callback() - { - internal_move(std::move(other)); - } -#endif - - enumerable_thread_specific& operator=( const enumerable_thread_specific& other ) - { - if( this != &other ) { - this->clear(); - my_construct_callback->destroy(); - internal_copy( other ); - } - return *this; - } - - template - enumerable_thread_specific& operator=( const enumerable_thread_specific& other ) - { - __TBB_ASSERT( static_cast(this)!=static_cast(&other), NULL ); // Objects of different types - this->clear(); - my_construct_callback->destroy(); - internal_copy(other); - return *this; - } - -#if __TBB_ETS_USE_CPP11 - enumerable_thread_specific& operator=( enumerable_thread_specific&& other ) - { - if( this != &other ) - internal_swap(other); - return *this; - } - - template - enumerable_thread_specific& operator=( enumerable_thread_specific&& other ) - { - __TBB_ASSERT( static_cast(this)!=static_cast(&other), NULL ); // Objects of different types - this->clear(); - my_construct_callback->destroy(); - internal_move(std::move(other)); - return *this; - } -#endif - - // combine_func_t has signature T(T,T) or T(const T&, const T&) - template - T combine(combine_func_t f_combine) { - if(begin() == end()) { - internal::ets_element location; - my_construct_callback->construct(location.value()); - return *location.value_committed(); - } - const_iterator ci = begin(); - T my_result = *ci; - while(++ci != end()) - my_result = f_combine( my_result, *ci ); - return my_result; - } - - // combine_func_t takes T by value or by [const] reference, and returns nothing - template - void combine_each(combine_func_t f_combine) { - for(iterator ci = begin(); ci != end(); ++ci) { - f_combine( *ci ); - } - } - - }; // enumerable_thread_specific - - template< typename Container > - class flattened2d { - - // This intermediate typedef is to address issues with VC7.1 compilers - typedef typename Container::value_type conval_type; - - public: - - //! Basic types - typedef typename conval_type::size_type size_type; - typedef typename conval_type::difference_type difference_type; - typedef typename conval_type::allocator_type allocator_type; - typedef typename conval_type::value_type value_type; - typedef typename conval_type::reference reference; - typedef typename conval_type::const_reference const_reference; - typedef typename conval_type::pointer pointer; - typedef typename conval_type::const_pointer const_pointer; - - typedef typename internal::segmented_iterator iterator; - typedef typename internal::segmented_iterator const_iterator; - - flattened2d( const Container &c, typename Container::const_iterator b, typename Container::const_iterator e ) : - my_container(const_cast(&c)), my_begin(b), my_end(e) { } - - explicit flattened2d( const Container &c ) : - my_container(const_cast(&c)), my_begin(c.begin()), my_end(c.end()) { } - - iterator begin() { return iterator(*my_container) = my_begin; } - iterator end() { return iterator(*my_container) = my_end; } - const_iterator begin() const { return const_iterator(*my_container) = my_begin; } - const_iterator end() const { return const_iterator(*my_container) = my_end; } - - size_type size() const { - size_type tot_size = 0; - for(typename Container::const_iterator i = my_begin; i != my_end; ++i) { - tot_size += i->size(); - } - return tot_size; - } - - private: - - Container *my_container; - typename Container::const_iterator my_begin; - typename Container::const_iterator my_end; - - }; - - template - flattened2d flatten2d(const Container &c, const typename Container::const_iterator b, const typename Container::const_iterator e) { - return flattened2d(c, b, e); - } - - template - flattened2d flatten2d(const Container &c) { - return flattened2d(c); - } - -} // interface6 - -namespace internal { -using interface6::internal::segmented_iterator; -} - -using interface6::enumerable_thread_specific; -using interface6::flattened2d; -using interface6::flatten2d; - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_enumerable_thread_specific_H_include_area - -#endif +#include "../oneapi/tbb/enumerable_thread_specific.h" diff --git a/src/3rd party/tbb/flow_graph.h b/src/3rd party/tbb/flow_graph.h index 746589c1f0..40da468fe0 100644 --- a/src/3rd party/tbb/flow_graph.h +++ b/src/3rd party/tbb/flow_graph.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,4335 +14,4 @@ limitations under the License. */ -#ifndef __TBB_flow_graph_H -#define __TBB_flow_graph_H - -#define __TBB_flow_graph_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "tbb_stddef.h" -#include "atomic.h" -#include "spin_mutex.h" -#include "null_mutex.h" -#include "spin_rw_mutex.h" -#include "null_rw_mutex.h" -#include "task.h" -#include "cache_aligned_allocator.h" -#include "tbb_exception.h" -#include "internal/_template_helpers.h" -#include "internal/_aggregator_impl.h" -#include "tbb_profiling.h" -#include "task_arena.h" - -#if TBB_USE_THREADING_TOOLS && TBB_PREVIEW_FLOW_GRAPH_TRACE && ( __linux__ || __APPLE__ ) - #if __INTEL_COMPILER - // Disabled warning "routine is both inline and noinline" - #pragma warning (push) - #pragma warning( disable: 2196 ) - #endif - #define __TBB_NOINLINE_SYM __attribute__((noinline)) -#else - #define __TBB_NOINLINE_SYM -#endif - -#if __TBB_PREVIEW_ASYNC_MSG -#include // std::vector in internal::async_storage -#include // std::shared_ptr in async_msg -#endif - -#if __TBB_PREVIEW_STREAMING_NODE -// For streaming_node -#include // std::array -#include // std::unordered_map -#include // std::decay, std::true_type, std::false_type -#endif // __TBB_PREVIEW_STREAMING_NODE - -#if TBB_DEPRECATED_FLOW_ENQUEUE -#define FLOW_SPAWN(a) tbb::task::enqueue((a)) -#else -#define FLOW_SPAWN(a) tbb::task::spawn((a)) -#endif - -// use the VC10 or gcc version of tuple if it is available. -#if __TBB_CPP11_TUPLE_PRESENT - #include -namespace tbb { - namespace flow { - using std::tuple; - using std::tuple_size; - using std::tuple_element; - using std::get; - } -} -#else - #include "compat/tuple" -#endif - -#include -#include - -/** @file - \brief The graph related classes and functions - - There are some applications that best express dependencies as messages - passed between nodes in a graph. These messages may contain data or - simply act as signals that a predecessors has completed. The graph - class and its associated node classes can be used to express such - applications. -*/ - -namespace tbb { -namespace flow { - -//! An enumeration the provides the two most common concurrency levels: unlimited and serial -enum concurrency { unlimited = 0, serial = 1 }; - -namespace interface11 { - -//! A generic null type -struct null_type {}; - -//! An empty class used for messages that mean "I'm done" -class continue_msg {}; - -//! Forward declaration section -template< typename T > class sender; -template< typename T > class receiver; -class continue_receiver; - -template< typename T, typename U > class limiter_node; // needed for resetting decrementer - -template< typename R, typename B > class run_and_put_task; - -namespace internal { - -template class successor_cache; -template class broadcast_cache; -template class round_robin_cache; -template class predecessor_cache; -template class reservable_predecessor_cache; - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET -namespace order { -struct following; -struct preceding; -} -template struct node_set; -#endif - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION -// Holder of edges both for caches and for those nodes which do not have predecessor caches. -// C == receiver< ... > or sender< ... >, depending. -template -class edge_container { - -public: - typedef std::list > edge_list_type; - - void add_edge(C &s) { - built_edges.push_back(&s); - } - - void delete_edge(C &s) { - for (typename edge_list_type::iterator i = built_edges.begin(); i != built_edges.end(); ++i) { - if (*i == &s) { - (void)built_edges.erase(i); - return; // only remove one predecessor per request - } - } - } - - void copy_edges(edge_list_type &v) { - v = built_edges; - } - - size_t edge_count() { - return (size_t)(built_edges.size()); - } - - void clear() { - built_edges.clear(); - } - - // methods remove the statement from all predecessors/successors liste in the edge - // container. - template< typename S > void sender_extract(S &s); - template< typename R > void receiver_extract(R &r); - -private: - edge_list_type built_edges; -}; // class edge_container -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - -} // namespace internal - -} // namespace interfaceX -} // namespace flow -} // namespace tbb - -//! The graph class -#include "internal/_flow_graph_impl.h" - -namespace tbb { -namespace flow { -namespace interface11 { - -// enqueue left task if necessary. Returns the non-enqueued task if there is one. -static inline tbb::task *combine_tasks(graph& g, tbb::task * left, tbb::task * right) { - // if no RHS task, don't change left. - if (right == NULL) return left; - // right != NULL - if (left == NULL) return right; - if (left == SUCCESSFULLY_ENQUEUED) return right; - // left contains a task - if (right != SUCCESSFULLY_ENQUEUED) { - // both are valid tasks - internal::spawn_in_graph_arena(g, *left); - return right; - } - return left; -} - -#if __TBB_PREVIEW_ASYNC_MSG - -template < typename T > class async_msg; - -namespace internal { - -template < typename T > class async_storage; - -template< typename T, typename = void > -struct async_helpers { - typedef async_msg async_type; - typedef T filtered_type; - - static const bool is_async_type = false; - - static const void* to_void_ptr(const T& t) { - return static_cast(&t); - } - - static void* to_void_ptr(T& t) { - return static_cast(&t); - } - - static const T& from_void_ptr(const void* p) { - return *static_cast(p); - } - - static T& from_void_ptr(void* p) { - return *static_cast(p); - } - - static task* try_put_task_wrapper_impl(receiver* const this_recv, const void *p, bool is_async) { - if (is_async) { - // This (T) is NOT async and incoming 'A t' IS async - // Get data from async_msg - const async_msg& msg = async_helpers< async_msg >::from_void_ptr(p); - task* const new_task = msg.my_storage->subscribe(*this_recv, this_recv->graph_reference()); - // finalize() must be called after subscribe() because set() can be called in finalize() - // and 'this_recv' client must be subscribed by this moment - msg.finalize(); - return new_task; - } - else { - // Incoming 't' is NOT async - return this_recv->try_put_task(from_void_ptr(p)); - } - } -}; - -template< typename T > -struct async_helpers< T, typename std::enable_if< std::is_base_of, T>::value >::type > { - typedef T async_type; - typedef typename T::async_msg_data_type filtered_type; - - static const bool is_async_type = true; - - // Receiver-classes use const interfaces - static const void* to_void_ptr(const T& t) { - return static_cast(&static_cast&>(t)); - } - - static void* to_void_ptr(T& t) { - return static_cast(&static_cast&>(t)); - } - - // Sender-classes use non-const interfaces - static const T& from_void_ptr(const void* p) { - return *static_cast(static_cast*>(p)); - } - - static T& from_void_ptr(void* p) { - return *static_cast(static_cast*>(p)); - } - - // Used in receiver class - static task* try_put_task_wrapper_impl(receiver* const this_recv, const void *p, bool is_async) { - if (is_async) { - // Both are async - return this_recv->try_put_task(from_void_ptr(p)); - } - else { - // This (T) is async and incoming 'X t' is NOT async - // Create async_msg for X - const filtered_type& t = async_helpers::from_void_ptr(p); - const T msg(t); - return this_recv->try_put_task(msg); - } - } -}; - -class untyped_receiver; - -class untyped_sender { - template< typename, typename > friend class internal::predecessor_cache; - template< typename, typename > friend class internal::reservable_predecessor_cache; -public: - //! The successor type for this node - typedef untyped_receiver successor_type; - - virtual ~untyped_sender() {} - - // NOTE: Following part of PUBLIC section is copy-paste from original sender class - - // TODO: Prevent untyped successor registration - - //! Add a new successor to this node - virtual bool register_successor( successor_type &r ) = 0; - - //! Removes a successor from this node - virtual bool remove_successor( successor_type &r ) = 0; - - //! Releases the reserved item - virtual bool try_release( ) { return false; } - - //! Consumes the reserved item - virtual bool try_consume( ) { return false; } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - //! interface to record edges for traversal & deletion - typedef internal::edge_container built_successors_type; - typedef built_successors_type::edge_list_type successor_list_type; - virtual built_successors_type &built_successors() = 0; - virtual void internal_add_built_successor( successor_type & ) = 0; - virtual void internal_delete_built_successor( successor_type & ) = 0; - virtual void copy_successors( successor_list_type &) = 0; - virtual size_t successor_count() = 0; -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ -protected: - //! Request an item from the sender - template< typename X > - bool try_get( X &t ) { - return try_get_wrapper( internal::async_helpers::to_void_ptr(t), internal::async_helpers::is_async_type ); - } - - //! Reserves an item in the sender - template< typename X > - bool try_reserve( X &t ) { - return try_reserve_wrapper( internal::async_helpers::to_void_ptr(t), internal::async_helpers::is_async_type ); - } - - virtual bool try_get_wrapper( void* p, bool is_async ) = 0; - virtual bool try_reserve_wrapper( void* p, bool is_async ) = 0; -}; - -class untyped_receiver { - template< typename, typename > friend class run_and_put_task; - - template< typename, typename > friend class internal::broadcast_cache; - template< typename, typename > friend class internal::round_robin_cache; - template< typename, typename > friend class internal::successor_cache; - -#if __TBB_PREVIEW_OPENCL_NODE - template< typename, typename > friend class proxy_dependency_receiver; -#endif /* __TBB_PREVIEW_OPENCL_NODE */ -public: - //! The predecessor type for this node - typedef untyped_sender predecessor_type; - - //! Destructor - virtual ~untyped_receiver() {} - - //! Put an item to the receiver - template - bool try_put(const X& t) { - task *res = try_put_task(t); - if (!res) return false; - if (res != SUCCESSFULLY_ENQUEUED) internal::spawn_in_graph_arena(graph_reference(), *res); - return true; - } - - // NOTE: Following part of PUBLIC section is copy-paste from original receiver class - - // TODO: Prevent untyped predecessor registration - - //! Add a predecessor to the node - virtual bool register_predecessor( predecessor_type & ) { return false; } - - //! Remove a predecessor from the node - virtual bool remove_predecessor( predecessor_type & ) { return false; } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef internal::edge_container built_predecessors_type; - typedef built_predecessors_type::edge_list_type predecessor_list_type; - virtual built_predecessors_type &built_predecessors() = 0; - virtual void internal_add_built_predecessor( predecessor_type & ) = 0; - virtual void internal_delete_built_predecessor( predecessor_type & ) = 0; - virtual void copy_predecessors( predecessor_list_type & ) = 0; - virtual size_t predecessor_count() = 0; -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ -protected: - template - task *try_put_task(const X& t) { - return try_put_task_wrapper( internal::async_helpers::to_void_ptr(t), internal::async_helpers::is_async_type ); - } - - virtual task* try_put_task_wrapper( const void* p, bool is_async ) = 0; - - virtual graph& graph_reference() const = 0; - - // NOTE: Following part of PROTECTED and PRIVATE sections is copy-paste from original receiver class - - //! put receiver back in initial state - virtual void reset_receiver(reset_flags f = rf_reset_protocol) = 0; - - virtual bool is_continue_receiver() { return false; } -}; - -} // namespace internal - -//! Pure virtual template class that defines a sender of messages of type T -template< typename T > -class sender : public internal::untyped_sender { -public: - //! The output type of this sender - __TBB_DEPRECATED typedef T output_type; - - __TBB_DEPRECATED typedef typename internal::async_helpers::filtered_type filtered_type; - - //! Request an item from the sender - virtual bool try_get( T & ) { return false; } - - //! Reserves an item in the sender - virtual bool try_reserve( T & ) { return false; } - -protected: - virtual bool try_get_wrapper( void* p, bool is_async ) __TBB_override { - // Both async OR both are NOT async - if ( internal::async_helpers::is_async_type == is_async ) { - return try_get( internal::async_helpers::from_void_ptr(p) ); - } - // Else: this (T) is async OR incoming 't' is async - __TBB_ASSERT(false, "async_msg interface does not support 'pull' protocol in try_get()"); - return false; - } - - virtual bool try_reserve_wrapper( void* p, bool is_async ) __TBB_override { - // Both async OR both are NOT async - if ( internal::async_helpers::is_async_type == is_async ) { - return try_reserve( internal::async_helpers::from_void_ptr(p) ); - } - // Else: this (T) is async OR incoming 't' is async - __TBB_ASSERT(false, "async_msg interface does not support 'pull' protocol in try_reserve()"); - return false; - } -}; // class sender - -//! Pure virtual template class that defines a receiver of messages of type T -template< typename T > -class receiver : public internal::untyped_receiver { - template< typename > friend class internal::async_storage; - template< typename, typename > friend struct internal::async_helpers; -public: - //! The input type of this receiver - __TBB_DEPRECATED typedef T input_type; - - __TBB_DEPRECATED typedef typename internal::async_helpers::filtered_type filtered_type; - - //! Put an item to the receiver - bool try_put( const typename internal::async_helpers::filtered_type& t ) { - return internal::untyped_receiver::try_put(t); - } - - bool try_put( const typename internal::async_helpers::async_type& t ) { - return internal::untyped_receiver::try_put(t); - } - -protected: - virtual task* try_put_task_wrapper( const void *p, bool is_async ) __TBB_override { - return internal::async_helpers::try_put_task_wrapper_impl(this, p, is_async); - } - - //! Put item to successor; return task to run the successor if possible. - virtual task *try_put_task(const T& t) = 0; - -}; // class receiver - -#else // __TBB_PREVIEW_ASYNC_MSG - -//! Pure virtual template class that defines a sender of messages of type T -template< typename T > -class sender { -public: - //! The output type of this sender - __TBB_DEPRECATED typedef T output_type; - - //! The successor type for this node - __TBB_DEPRECATED typedef receiver successor_type; - - virtual ~sender() {} - - // NOTE: Following part of PUBLIC section is partly copy-pasted in sender under #if __TBB_PREVIEW_ASYNC_MSG - - //! Add a new successor to this node - __TBB_DEPRECATED virtual bool register_successor( successor_type &r ) = 0; - - //! Removes a successor from this node - __TBB_DEPRECATED virtual bool remove_successor( successor_type &r ) = 0; - - //! Request an item from the sender - virtual bool try_get( T & ) { return false; } - - //! Reserves an item in the sender - virtual bool try_reserve( T & ) { return false; } - - //! Releases the reserved item - virtual bool try_release( ) { return false; } - - //! Consumes the reserved item - virtual bool try_consume( ) { return false; } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - //! interface to record edges for traversal & deletion - __TBB_DEPRECATED typedef typename internal::edge_container built_successors_type; - __TBB_DEPRECATED typedef typename built_successors_type::edge_list_type successor_list_type; - __TBB_DEPRECATED virtual built_successors_type &built_successors() = 0; - __TBB_DEPRECATED virtual void internal_add_built_successor( successor_type & ) = 0; - __TBB_DEPRECATED virtual void internal_delete_built_successor( successor_type & ) = 0; - __TBB_DEPRECATED virtual void copy_successors( successor_list_type &) = 0; - __TBB_DEPRECATED virtual size_t successor_count() = 0; -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ -}; // class sender - -//! Pure virtual template class that defines a receiver of messages of type T -template< typename T > -class receiver { -public: - //! The input type of this receiver - __TBB_DEPRECATED typedef T input_type; - - //! The predecessor type for this node - __TBB_DEPRECATED typedef sender predecessor_type; - - //! Destructor - virtual ~receiver() {} - - //! Put an item to the receiver - bool try_put( const T& t ) { - task *res = try_put_task(t); - if (!res) return false; - if (res != SUCCESSFULLY_ENQUEUED) internal::spawn_in_graph_arena(graph_reference(), *res); - return true; - } - - //! put item to successor; return task to run the successor if possible. -protected: - template< typename R, typename B > friend class run_and_put_task; - template< typename X, typename Y > friend class internal::broadcast_cache; - template< typename X, typename Y > friend class internal::round_robin_cache; - virtual task *try_put_task(const T& t) = 0; - virtual graph& graph_reference() const = 0; -public: - // NOTE: Following part of PUBLIC and PROTECTED sections is copy-pasted in receiver under #if __TBB_PREVIEW_ASYNC_MSG - - //! Add a predecessor to the node - __TBB_DEPRECATED virtual bool register_predecessor( predecessor_type & ) { return false; } - - //! Remove a predecessor from the node - __TBB_DEPRECATED virtual bool remove_predecessor( predecessor_type & ) { return false; } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - __TBB_DEPRECATED typedef typename internal::edge_container built_predecessors_type; - __TBB_DEPRECATED typedef typename built_predecessors_type::edge_list_type predecessor_list_type; - __TBB_DEPRECATED virtual built_predecessors_type &built_predecessors() = 0; - __TBB_DEPRECATED virtual void internal_add_built_predecessor( predecessor_type & ) = 0; - __TBB_DEPRECATED virtual void internal_delete_built_predecessor( predecessor_type & ) = 0; - __TBB_DEPRECATED virtual void copy_predecessors( predecessor_list_type & ) = 0; - __TBB_DEPRECATED virtual size_t predecessor_count() = 0; -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - -protected: - //! put receiver back in initial state - virtual void reset_receiver(reset_flags f = rf_reset_protocol) = 0; - - template friend class internal::successor_cache; - virtual bool is_continue_receiver() { return false; } - -#if __TBB_PREVIEW_OPENCL_NODE - template< typename, typename > friend class proxy_dependency_receiver; -#endif /* __TBB_PREVIEW_OPENCL_NODE */ -}; // class receiver - -#endif // __TBB_PREVIEW_ASYNC_MSG - -//! Base class for receivers of completion messages -/** These receivers automatically reset, but cannot be explicitly waited on */ -class continue_receiver : public receiver< continue_msg > { -public: - - //! The input type - __TBB_DEPRECATED typedef continue_msg input_type; - - //! The predecessor type for this node - __TBB_DEPRECATED typedef receiver::predecessor_type predecessor_type; - - //! Constructor - __TBB_DEPRECATED explicit continue_receiver( - __TBB_FLOW_GRAPH_PRIORITY_ARG1(int number_of_predecessors, node_priority_t priority)) { - my_predecessor_count = my_initial_predecessor_count = number_of_predecessors; - my_current_count = 0; - __TBB_FLOW_GRAPH_PRIORITY_EXPR( my_priority = priority; ) - } - - //! Copy constructor - __TBB_DEPRECATED continue_receiver( const continue_receiver& src ) : receiver() { - my_predecessor_count = my_initial_predecessor_count = src.my_initial_predecessor_count; - my_current_count = 0; - __TBB_FLOW_GRAPH_PRIORITY_EXPR( my_priority = src.my_priority; ) - } - - //! Increments the trigger threshold - __TBB_DEPRECATED bool register_predecessor( predecessor_type & ) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - ++my_predecessor_count; - return true; - } - - //! Decrements the trigger threshold - /** Does not check to see if the removal of the predecessor now makes the current count - exceed the new threshold. So removing a predecessor while the graph is active can cause - unexpected results. */ - __TBB_DEPRECATED bool remove_predecessor( predecessor_type & ) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - --my_predecessor_count; - return true; - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - __TBB_DEPRECATED typedef internal::edge_container built_predecessors_type; - __TBB_DEPRECATED typedef built_predecessors_type::edge_list_type predecessor_list_type; - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - __TBB_DEPRECATED void internal_add_built_predecessor( predecessor_type &s) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - my_built_predecessors.add_edge( s ); - } - - __TBB_DEPRECATED void internal_delete_built_predecessor( predecessor_type &s) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - my_built_predecessors.delete_edge(s); - } - - __TBB_DEPRECATED void copy_predecessors( predecessor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - my_built_predecessors.copy_edges(v); - } - - __TBB_DEPRECATED size_t predecessor_count() __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - return my_built_predecessors.edge_count(); - } - -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - // execute body is supposed to be too small to create a task for. - task *try_put_task( const input_type & ) __TBB_override { - { - spin_mutex::scoped_lock l(my_mutex); - if ( ++my_current_count < my_predecessor_count ) - return SUCCESSFULLY_ENQUEUED; - else - my_current_count = 0; - } - task * res = execute(); - return res? res : SUCCESSFULLY_ENQUEUED; - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - // continue_receiver must contain its own built_predecessors because it does - // not have a node_cache. - built_predecessors_type my_built_predecessors; -#endif - spin_mutex my_mutex; - int my_predecessor_count; - int my_current_count; - int my_initial_predecessor_count; - __TBB_FLOW_GRAPH_PRIORITY_EXPR( node_priority_t my_priority; ) - // the friend declaration in the base class did not eliminate the "protected class" - // error in gcc 4.1.2 - template friend class tbb::flow::interface11::limiter_node; - - void reset_receiver( reset_flags f ) __TBB_override { - my_current_count = 0; - if (f & rf_clear_edges) { -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - my_built_predecessors.clear(); -#endif - my_predecessor_count = my_initial_predecessor_count; - } - } - - //! Does whatever should happen when the threshold is reached - /** This should be very fast or else spawn a task. This is - called while the sender is blocked in the try_put(). */ - virtual task * execute() = 0; - template friend class internal::successor_cache; - bool is_continue_receiver() __TBB_override { return true; } - -}; // class continue_receiver - -} // interfaceX - -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - template - K key_from_message( const T &t ) { - return t.key(); - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - - using interface11::sender; - using interface11::receiver; - using interface11::continue_receiver; -} // flow -} // tbb - -#include "internal/_flow_graph_trace_impl.h" -#include "internal/_tbb_hash_compare_impl.h" - -namespace tbb { -namespace flow { -namespace interface11 { - -#include "internal/_flow_graph_body_impl.h" -#include "internal/_flow_graph_cache_impl.h" -#include "internal/_flow_graph_types_impl.h" -#if __TBB_PREVIEW_ASYNC_MSG -#include "internal/_flow_graph_async_msg_impl.h" -#endif -using namespace internal::graph_policy_namespace; - -template -graph_iterator::graph_iterator(C *g, bool begin) : my_graph(g), current_node(NULL) -{ - if (begin) current_node = my_graph->my_nodes; - //else it is an end iterator by default -} - -template -typename graph_iterator::reference graph_iterator::operator*() const { - __TBB_ASSERT(current_node, "graph_iterator at end"); - return *operator->(); -} - -template -typename graph_iterator::pointer graph_iterator::operator->() const { - return current_node; -} - -template -void graph_iterator::internal_forward() { - if (current_node) current_node = current_node->next; -} - -} // namespace interfaceX - -namespace interface10 { -//! Constructs a graph with isolated task_group_context -inline graph::graph() : my_nodes(NULL), my_nodes_last(NULL), my_task_arena(NULL) { - prepare_task_arena(); - own_context = true; - cancelled = false; - caught_exception = false; - my_context = new task_group_context(tbb::internal::FLOW_TASKS); - my_root_task = (new (task::allocate_root(*my_context)) empty_task); - my_root_task->set_ref_count(1); - tbb::internal::fgt_graph(this); - my_is_active = true; -} - -inline graph::graph(task_group_context& use_this_context) : - my_context(&use_this_context), my_nodes(NULL), my_nodes_last(NULL), my_task_arena(NULL) { - prepare_task_arena(); - own_context = false; - cancelled = false; - caught_exception = false; - my_root_task = (new (task::allocate_root(*my_context)) empty_task); - my_root_task->set_ref_count(1); - tbb::internal::fgt_graph(this); - my_is_active = true; -} - -inline graph::~graph() { - wait_for_all(); - my_root_task->set_ref_count(0); - tbb::task::destroy(*my_root_task); - if (own_context) delete my_context; - delete my_task_arena; -} - -inline void graph::reserve_wait() { - if (my_root_task) { - my_root_task->increment_ref_count(); - tbb::internal::fgt_reserve_wait(this); - } -} - -inline void graph::release_wait() { - if (my_root_task) { - tbb::internal::fgt_release_wait(this); - my_root_task->decrement_ref_count(); - } -} - -inline void graph::register_node(tbb::flow::interface11::graph_node *n) { - n->next = NULL; - { - spin_mutex::scoped_lock lock(nodelist_mutex); - n->prev = my_nodes_last; - if (my_nodes_last) my_nodes_last->next = n; - my_nodes_last = n; - if (!my_nodes) my_nodes = n; - } -} - -inline void graph::remove_node(tbb::flow::interface11::graph_node *n) { - { - spin_mutex::scoped_lock lock(nodelist_mutex); - __TBB_ASSERT(my_nodes && my_nodes_last, "graph::remove_node: Error: no registered nodes"); - if (n->prev) n->prev->next = n->next; - if (n->next) n->next->prev = n->prev; - if (my_nodes_last == n) my_nodes_last = n->prev; - if (my_nodes == n) my_nodes = n->next; - } - n->prev = n->next = NULL; -} - -inline void graph::reset( tbb::flow::interface11::reset_flags f ) { - // reset context - tbb::flow::interface11::internal::deactivate_graph(*this); - - if(my_context) my_context->reset(); - cancelled = false; - caught_exception = false; - // reset all the nodes comprising the graph - for(iterator ii = begin(); ii != end(); ++ii) { - tbb::flow::interface11::graph_node *my_p = &(*ii); - my_p->reset_node(f); - } - // Reattach the arena. Might be useful to run the graph in a particular task_arena - // while not limiting graph lifetime to a single task_arena::execute() call. - prepare_task_arena( /*reinit=*/true ); - tbb::flow::interface11::internal::activate_graph(*this); - // now spawn the tasks necessary to start the graph - for(task_list_type::iterator rti = my_reset_task_list.begin(); rti != my_reset_task_list.end(); ++rti) { - tbb::flow::interface11::internal::spawn_in_graph_arena(*this, *(*rti)); - } - my_reset_task_list.clear(); -} - -inline graph::iterator graph::begin() { return iterator(this, true); } - -inline graph::iterator graph::end() { return iterator(this, false); } - -inline graph::const_iterator graph::begin() const { return const_iterator(this, true); } - -inline graph::const_iterator graph::end() const { return const_iterator(this, false); } - -inline graph::const_iterator graph::cbegin() const { return const_iterator(this, true); } - -inline graph::const_iterator graph::cend() const { return const_iterator(this, false); } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE -inline void graph::set_name(const char *name) { - tbb::internal::fgt_graph_desc(this, name); -} -#endif - -} // namespace interface10 - -namespace interface11 { - -inline graph_node::graph_node(graph& g) : my_graph(g) { - my_graph.register_node(this); -} - -inline graph_node::~graph_node() { - my_graph.remove_node(this); -} - -#include "internal/_flow_graph_node_impl.h" - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET -using internal::node_set; -#endif - -//! An executable node that acts as a source, i.e. it has no predecessors -template < typename Output > -class source_node : public graph_node, public sender< Output > { -public: - //! The type of the output message, which is complete - typedef Output output_type; - - //! The type of successors of this node - typedef typename sender::successor_type successor_type; - - //Source node has no input type - typedef null_type input_type; - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename sender::built_successors_type built_successors_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - - //! Constructor for a node with a successor - template< typename Body > - __TBB_NOINLINE_SYM source_node( graph &g, Body body, bool is_active = true ) - : graph_node(g), my_active(is_active), init_my_active(is_active), - my_body( new internal::source_body_leaf< output_type, Body>(body) ), - my_init_body( new internal::source_body_leaf< output_type, Body>(body) ), - my_reserved(false), my_has_cached_item(false) - { - my_successors.set_owner(this); - tbb::internal::fgt_node_with_body( CODEPTR(), tbb::internal::FLOW_SOURCE_NODE, &this->my_graph, - static_cast *>(this), this->my_body ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - source_node( const node_set& successors, Body body, bool is_active = true ) - : source_node(successors.graph_reference(), body, is_active) { - make_edges(*this, successors); - } -#endif - - //! Copy constructor - __TBB_NOINLINE_SYM source_node( const source_node& src ) : - graph_node(src.my_graph), sender(), - my_active(src.init_my_active), - init_my_active(src.init_my_active), my_body( src.my_init_body->clone() ), my_init_body(src.my_init_body->clone() ), - my_reserved(false), my_has_cached_item(false) - { - my_successors.set_owner(this); - tbb::internal::fgt_node_with_body(CODEPTR(), tbb::internal::FLOW_SOURCE_NODE, &this->my_graph, - static_cast *>(this), this->my_body ); - } - - //! The destructor - ~source_node() { delete my_body; delete my_init_body; } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - //! Add a new successor to this node - bool register_successor( successor_type &r ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - my_successors.register_successor(r); - if ( my_active ) - spawn_put(); - return true; - } - - //! Removes a successor from this node - bool remove_successor( successor_type &r ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - my_successors.remove_successor(r); - return true; - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - void internal_add_built_successor( successor_type &r) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - my_successors.internal_add_built_successor(r); - } - - void internal_delete_built_successor( successor_type &r) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - my_successors.internal_delete_built_successor(r); - } - - size_t successor_count() __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - return my_successors.successor_count(); - } - - void copy_successors(successor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l(my_mutex); - my_successors.copy_successors(v); - } -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - - //! Request an item from the node - bool try_get( output_type &v ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - if ( my_reserved ) - return false; - - if ( my_has_cached_item ) { - v = my_cached_item; - my_has_cached_item = false; - return true; - } - // we've been asked to provide an item, but we have none. enqueue a task to - // provide one. - spawn_put(); - return false; - } - - //! Reserves an item. - bool try_reserve( output_type &v ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - if ( my_reserved ) { - return false; - } - - if ( my_has_cached_item ) { - v = my_cached_item; - my_reserved = true; - return true; - } else { - return false; - } - } - - //! Release a reserved item. - /** true = item has been released and so remains in sender, dest must request or reserve future items */ - bool try_release( ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - __TBB_ASSERT( my_reserved && my_has_cached_item, "releasing non-existent reservation" ); - my_reserved = false; - if(!my_successors.empty()) - spawn_put(); - return true; - } - - //! Consumes a reserved item - bool try_consume( ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - __TBB_ASSERT( my_reserved && my_has_cached_item, "consuming non-existent reservation" ); - my_reserved = false; - my_has_cached_item = false; - if ( !my_successors.empty() ) { - spawn_put(); - } - return true; - } - - //! Activates a node that was created in the inactive state - void activate() { - spin_mutex::scoped_lock lock(my_mutex); - my_active = true; - if (!my_successors.empty()) - spawn_put(); - } - - template - Body copy_function_object() { - internal::source_body &body_ref = *this->my_body; - return dynamic_cast< internal::source_body_leaf & >(body_ref).get_body(); - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - void extract( ) __TBB_override { - my_successors.built_successors().sender_extract(*this); // removes "my_owner" == this from each successor - my_active = init_my_active; - my_reserved = false; - if(my_has_cached_item) my_has_cached_item = false; - } -#endif - -protected: - - //! resets the source_node to its initial state - void reset_node( reset_flags f) __TBB_override { - my_active = init_my_active; - my_reserved =false; - if(my_has_cached_item) { - my_has_cached_item = false; - } - if(f & rf_clear_edges) my_successors.clear(); - if(f & rf_reset_bodies) { - internal::source_body *tmp = my_init_body->clone(); - delete my_body; - my_body = tmp; - } - if(my_active) - internal::add_task_to_graph_reset_list(this->my_graph, create_put_task()); - } - -private: - spin_mutex my_mutex; - bool my_active; - bool init_my_active; - internal::source_body *my_body; - internal::source_body *my_init_body; - internal::broadcast_cache< output_type > my_successors; - bool my_reserved; - bool my_has_cached_item; - output_type my_cached_item; - - // used by apply_body_bypass, can invoke body of node. - bool try_reserve_apply_body(output_type &v) { - spin_mutex::scoped_lock lock(my_mutex); - if ( my_reserved ) { - return false; - } - if ( !my_has_cached_item ) { - tbb::internal::fgt_begin_body( my_body ); - bool r = (*my_body)(my_cached_item); - tbb::internal::fgt_end_body( my_body ); - if (r) { - my_has_cached_item = true; - } - } - if ( my_has_cached_item ) { - v = my_cached_item; - my_reserved = true; - return true; - } else { - return false; - } - } - - // when resetting, and if the source_node was created with my_active == true, then - // when we reset the node we must store a task to run the node, and spawn it only - // after the reset is complete and is_active() is again true. This is why we don't - // test for is_active() here. - task* create_put_task() { - return ( new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal:: source_task_bypass < source_node< output_type > >( *this ) ); - } - - //! Spawns a task that applies the body - void spawn_put( ) { - if(internal::is_graph_active(this->my_graph)) { - internal::spawn_in_graph_arena(this->my_graph, *create_put_task()); - } - } - - friend class internal::source_task_bypass< source_node< output_type > >; - //! Applies the body. Returning SUCCESSFULLY_ENQUEUED okay; forward_task_bypass will handle it. - task * apply_body_bypass( ) { - output_type v; - if ( !try_reserve_apply_body(v) ) - return NULL; - - task *last_task = my_successors.try_put_task(v); - if ( last_task ) - try_consume(); - else - try_release(); - return last_task; - } -}; // class source_node - -//! Implements a function node that supports Input -> Output -template < typename Input, typename Output = continue_msg, typename Policy = queueing, typename Allocator=cache_aligned_allocator > -class function_node : public graph_node, public internal::function_input, public internal::function_output { -public: - typedef Input input_type; - typedef Output output_type; - typedef internal::function_input input_impl_type; - typedef internal::function_input_queue input_queue_type; - typedef internal::function_output fOutput_type; - typedef typename input_impl_type::predecessor_type predecessor_type; - typedef typename fOutput_type::successor_type successor_type; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename input_impl_type::predecessor_list_type predecessor_list_type; - typedef typename fOutput_type::successor_list_type successor_list_type; -#endif - using input_impl_type::my_predecessors; - - //! Constructor - // input_queue_type is allocated here, but destroyed in the function_input_base. - // TODO: pass the graph_buffer_policy to the function_input_base so it can all - // be done in one place. This would be an interface-breaking change. - template< typename Body > - __TBB_NOINLINE_SYM function_node( graph &g, size_t concurrency, -#if __TBB_CPP11_PRESENT - Body body, __TBB_FLOW_GRAPH_PRIORITY_ARG1( Policy = Policy(), node_priority_t priority = tbb::flow::internal::no_priority )) -#else - __TBB_FLOW_GRAPH_PRIORITY_ARG1( Body body, node_priority_t priority = tbb::flow::internal::no_priority )) -#endif - : graph_node(g), input_impl_type(g, concurrency, __TBB_FLOW_GRAPH_PRIORITY_ARG1(body, priority)), - fOutput_type(g) { - tbb::internal::fgt_node_with_body( CODEPTR(), tbb::internal::FLOW_FUNCTION_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this), this->my_body ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES && __TBB_CPP11_PRESENT - template - function_node( graph& g, size_t concurrency, Body body, node_priority_t priority ) - : function_node(g, concurrency, body, Policy(), priority) {} -#endif // __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES && __TBB_CPP11_PRESENT - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - function_node( const node_set& nodes, size_t concurrency, Body body, - __TBB_FLOW_GRAPH_PRIORITY_ARG1( Policy p = Policy(), node_priority_t priority = tbb::flow::internal::no_priority )) - : function_node(nodes.graph_reference(), concurrency, body, __TBB_FLOW_GRAPH_PRIORITY_ARG1(p, priority)) { - make_edges_in_order(nodes, *this); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES - template - function_node( const node_set& nodes, size_t concurrency, Body body, node_priority_t priority ) - : function_node(nodes, concurrency, body, Policy(), priority) {} -#endif // __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES -#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - - //! Copy constructor - __TBB_NOINLINE_SYM function_node( const function_node& src ) : - graph_node(src.my_graph), - input_impl_type(src), - fOutput_type(src.my_graph) { - tbb::internal::fgt_node_with_body( CODEPTR(), tbb::internal::FLOW_FUNCTION_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this), this->my_body ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - void extract( ) __TBB_override { - my_predecessors.built_predecessors().receiver_extract(*this); - successors().built_successors().sender_extract(*this); - } -#endif - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - using input_impl_type::try_put_task; - - internal::broadcast_cache &successors () __TBB_override { return fOutput_type::my_successors; } - - void reset_node(reset_flags f) __TBB_override { - input_impl_type::reset_function_input(f); - // TODO: use clear() instead. - if(f & rf_clear_edges) { - successors().clear(); - my_predecessors.clear(); - } - __TBB_ASSERT(!(f & rf_clear_edges) || successors().empty(), "function_node successors not empty"); - __TBB_ASSERT(this->my_predecessors.empty(), "function_node predecessors not empty"); - } - -}; // class function_node - -//! implements a function node that supports Input -> (set of outputs) -// Output is a tuple of output types. -template < typename Input, typename Output, typename Policy = queueing, typename Allocator=cache_aligned_allocator > -class multifunction_node : - public graph_node, - public internal::multifunction_input - < - Input, - typename internal::wrap_tuple_elements< - tbb::flow::tuple_size::value, // #elements in tuple - internal::multifunction_output, // wrap this around each element - Output // the tuple providing the types - >::type, - Policy, - Allocator - > { -protected: - static const int N = tbb::flow::tuple_size::value; -public: - typedef Input input_type; - typedef null_type output_type; - typedef typename internal::wrap_tuple_elements::type output_ports_type; - typedef internal::multifunction_input input_impl_type; - typedef internal::function_input_queue input_queue_type; -private: - typedef typename internal::multifunction_input base_type; - using input_impl_type::my_predecessors; -public: - template - __TBB_NOINLINE_SYM multifunction_node( - graph &g, size_t concurrency, -#if __TBB_CPP11_PRESENT - Body body, __TBB_FLOW_GRAPH_PRIORITY_ARG1( Policy = Policy(), node_priority_t priority = tbb::flow::internal::no_priority ) -#else - __TBB_FLOW_GRAPH_PRIORITY_ARG1(Body body, node_priority_t priority = tbb::flow::internal::no_priority) -#endif - ) : graph_node(g), base_type(g, concurrency, __TBB_FLOW_GRAPH_PRIORITY_ARG1(body, priority)) { - tbb::internal::fgt_multioutput_node_with_body( - CODEPTR(), tbb::internal::FLOW_MULTIFUNCTION_NODE, - &this->my_graph, static_cast *>(this), - this->output_ports(), this->my_body - ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES && __TBB_CPP11_PRESENT - template - __TBB_NOINLINE_SYM multifunction_node(graph& g, size_t concurrency, Body body, node_priority_t priority) - : multifunction_node(g, concurrency, body, Policy(), priority) {} -#endif // TBB_PREVIEW_FLOW_GRAPH_PRIORITIES && __TBB_CPP11_PRESENT - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - __TBB_NOINLINE_SYM multifunction_node(const node_set& nodes, size_t concurrency, Body body, - __TBB_FLOW_GRAPH_PRIORITY_ARG1(Policy p = Policy(), node_priority_t priority = tbb::flow::internal::no_priority)) - : multifunction_node(nodes.graph_reference(), concurrency, body, __TBB_FLOW_GRAPH_PRIORITY_ARG1(p, priority)) { - make_edges_in_order(nodes, *this); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES - template - __TBB_NOINLINE_SYM multifunction_node(const node_set& nodes, size_t concurrency, Body body, node_priority_t priority) - : multifunction_node(nodes, concurrency, body, Policy(), priority) {} -#endif // __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES -#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - - __TBB_NOINLINE_SYM multifunction_node( const multifunction_node &other) : - graph_node(other.my_graph), base_type(other) { - tbb::internal::fgt_multioutput_node_with_body( CODEPTR(), tbb::internal::FLOW_MULTIFUNCTION_NODE, - &this->my_graph, static_cast *>(this), - this->output_ports(), this->my_body ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multioutput_node_desc( this, name ); - } -#endif - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - void extract( ) __TBB_override { - my_predecessors.built_predecessors().receiver_extract(*this); - base_type::extract(); - } -#endif - // all the guts are in multifunction_input... -protected: - void reset_node(reset_flags f) __TBB_override { base_type::reset(f); } -}; // multifunction_node - -//! split_node: accepts a tuple as input, forwards each element of the tuple to its -// successors. The node has unlimited concurrency, so it does not reject inputs. -template > -class split_node : public graph_node, public receiver { - static const int N = tbb::flow::tuple_size::value; - typedef receiver base_type; -public: - typedef TupleType input_type; - typedef Allocator allocator_type; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename base_type::predecessor_type predecessor_type; - typedef typename base_type::predecessor_list_type predecessor_list_type; - typedef internal::predecessor_cache predecessor_cache_type; - typedef typename predecessor_cache_type::built_predecessors_type built_predecessors_type; -#endif - - typedef typename internal::wrap_tuple_elements< - N, // #elements in tuple - internal::multifunction_output, // wrap this around each element - TupleType // the tuple providing the types - >::type output_ports_type; - - __TBB_NOINLINE_SYM explicit split_node(graph &g) - : graph_node(g), - my_output_ports(internal::init_output_ports::call(g, my_output_ports)) - { - tbb::internal::fgt_multioutput_node(CODEPTR(), tbb::internal::FLOW_SPLIT_NODE, &this->my_graph, - static_cast *>(this), this->output_ports()); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - __TBB_NOINLINE_SYM split_node(const node_set& nodes) : split_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - __TBB_NOINLINE_SYM split_node(const split_node& other) - : graph_node(other.my_graph), base_type(other), - my_output_ports(internal::init_output_ports::call(other.my_graph, my_output_ports)) - { - tbb::internal::fgt_multioutput_node(CODEPTR(), tbb::internal::FLOW_SPLIT_NODE, &this->my_graph, - static_cast *>(this), this->output_ports()); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multioutput_node_desc( this, name ); - } -#endif - - output_ports_type &output_ports() { return my_output_ports; } - -protected: - task *try_put_task(const TupleType& t) __TBB_override { - // Sending split messages in parallel is not justified, as overheads would prevail. - // Also, we do not have successors here. So we just tell the task returned here is successful. - return internal::emit_element::emit_this(this->my_graph, t, output_ports()); - } - void reset_node(reset_flags f) __TBB_override { - if (f & rf_clear_edges) - internal::clear_element::clear_this(my_output_ports); - - __TBB_ASSERT(!(f & rf_clear_edges) || internal::clear_element::this_empty(my_output_ports), "split_node reset failed"); - } - void reset_receiver(reset_flags /*f*/) __TBB_override {} - graph& graph_reference() const __TBB_override { - return my_graph; - } -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION -private: //! split_node doesn't use this "predecessors" functionality; so, we have "dummies" here; - void extract() __TBB_override {} - - //! Adds to list of predecessors added by make_edge - void internal_add_built_predecessor(predecessor_type&) __TBB_override {} - - //! removes from to list of predecessors (used by remove_edge) - void internal_delete_built_predecessor(predecessor_type&) __TBB_override {} - - size_t predecessor_count() __TBB_override { return 0; } - - void copy_predecessors(predecessor_list_type&) __TBB_override {} - - built_predecessors_type &built_predecessors() __TBB_override { return my_predessors; } - - //! dummy member - built_predecessors_type my_predessors; -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - -private: - output_ports_type my_output_ports; -}; - -//! Implements an executable node that supports continue_msg -> Output -template > -class continue_node : public graph_node, public internal::continue_input, - public internal::function_output { -public: - typedef continue_msg input_type; - typedef Output output_type; - typedef internal::continue_input input_impl_type; - typedef internal::function_output fOutput_type; - typedef typename input_impl_type::predecessor_type predecessor_type; - typedef typename fOutput_type::successor_type successor_type; - - //! Constructor for executable node with continue_msg -> Output - template - __TBB_NOINLINE_SYM continue_node( - graph &g, -#if __TBB_CPP11_PRESENT - Body body, __TBB_FLOW_GRAPH_PRIORITY_ARG1( Policy = Policy(), node_priority_t priority = tbb::flow::internal::no_priority ) -#else - __TBB_FLOW_GRAPH_PRIORITY_ARG1( Body body, node_priority_t priority = tbb::flow::internal::no_priority ) -#endif - ) : graph_node(g), input_impl_type( g, __TBB_FLOW_GRAPH_PRIORITY_ARG1(body, priority) ), - fOutput_type(g) { - tbb::internal::fgt_node_with_body( CODEPTR(), tbb::internal::FLOW_CONTINUE_NODE, &this->my_graph, - - static_cast *>(this), - static_cast *>(this), this->my_body ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES && __TBB_CPP11_PRESENT - template - continue_node( graph& g, Body body, node_priority_t priority ) - : continue_node(g, body, Policy(), priority) {} -#endif - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - continue_node( const node_set& nodes, Body body, - __TBB_FLOW_GRAPH_PRIORITY_ARG1( Policy p = Policy(), node_priority_t priority = tbb::flow::internal::no_priority)) - : continue_node(nodes.graph_reference(), body, __TBB_FLOW_GRAPH_PRIORITY_ARG1(p, priority) ) { - make_edges_in_order(nodes, *this); - } -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES - template - continue_node( const node_set& nodes, Body body, node_priority_t priority) - : continue_node(nodes, body, Policy(), priority) {} -#endif // __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES -#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - - //! Constructor for executable node with continue_msg -> Output - template - __TBB_NOINLINE_SYM continue_node( - graph &g, int number_of_predecessors, -#if __TBB_CPP11_PRESENT - Body body, __TBB_FLOW_GRAPH_PRIORITY_ARG1( Policy = Policy(), node_priority_t priority = tbb::flow::internal::no_priority ) -#else - __TBB_FLOW_GRAPH_PRIORITY_ARG1( Body body, node_priority_t priority = tbb::flow::internal::no_priority ) -#endif - ) : graph_node(g) - , input_impl_type(g, number_of_predecessors, __TBB_FLOW_GRAPH_PRIORITY_ARG1(body, priority)), - fOutput_type(g) { - tbb::internal::fgt_node_with_body( CODEPTR(), tbb::internal::FLOW_CONTINUE_NODE, &this->my_graph, - static_cast *>(this), - static_cast *>(this), this->my_body ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES && __TBB_CPP11_PRESENT - template - continue_node( graph& g, int number_of_predecessors, Body body, node_priority_t priority) - : continue_node(g, number_of_predecessors, body, Policy(), priority) {} -#endif - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - continue_node( const node_set& nodes, int number_of_predecessors, - Body body, __TBB_FLOW_GRAPH_PRIORITY_ARG1( Policy p = Policy(), node_priority_t priority = tbb::flow::internal::no_priority )) - : continue_node(nodes.graph_reference(), number_of_predecessors, body, __TBB_FLOW_GRAPH_PRIORITY_ARG1(p, priority)) { - make_edges_in_order(nodes, *this); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES - template - continue_node( const node_set& nodes, int number_of_predecessors, - Body body, node_priority_t priority ) - : continue_node(nodes, number_of_predecessors, body, Policy(), priority) {} -#endif -#endif - - //! Copy constructor - __TBB_NOINLINE_SYM continue_node( const continue_node& src ) : - graph_node(src.my_graph), input_impl_type(src), - internal::function_output(src.my_graph) { - tbb::internal::fgt_node_with_body( CODEPTR(), tbb::internal::FLOW_CONTINUE_NODE, &this->my_graph, - static_cast *>(this), - static_cast *>(this), this->my_body ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - void extract() __TBB_override { - input_impl_type::my_built_predecessors.receiver_extract(*this); - successors().built_successors().sender_extract(*this); - } -#endif - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - using input_impl_type::try_put_task; - internal::broadcast_cache &successors () __TBB_override { return fOutput_type::my_successors; } - - void reset_node(reset_flags f) __TBB_override { - input_impl_type::reset_receiver(f); - if(f & rf_clear_edges)successors().clear(); - __TBB_ASSERT(!(f & rf_clear_edges) || successors().empty(), "continue_node not reset"); - } -}; // continue_node - -//! Forwards messages of type T to all successors -template -class broadcast_node : public graph_node, public receiver, public sender { -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename sender::successor_list_type successor_list_type; -#endif -private: - internal::broadcast_cache my_successors; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - internal::edge_container my_built_predecessors; - spin_mutex pred_mutex; // serialize accesses on edge_container -#endif -public: - - __TBB_NOINLINE_SYM explicit broadcast_node(graph& g) : graph_node(g) { - my_successors.set_owner( this ); - tbb::internal::fgt_node( CODEPTR(), tbb::internal::FLOW_BROADCAST_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - broadcast_node(const node_set& nodes) : broadcast_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - // Copy constructor - __TBB_NOINLINE_SYM broadcast_node( const broadcast_node& src ) : - graph_node(src.my_graph), receiver(), sender() - { - my_successors.set_owner( this ); - tbb::internal::fgt_node( CODEPTR(), tbb::internal::FLOW_BROADCAST_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - //! Adds a successor - bool register_successor( successor_type &r ) __TBB_override { - my_successors.register_successor( r ); - return true; - } - - //! Removes s as a successor - bool remove_successor( successor_type &r ) __TBB_override { - my_successors.remove_successor( r ); - return true; - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename sender::built_successors_type built_successors_type; - - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - void internal_add_built_successor(successor_type &r) __TBB_override { - my_successors.internal_add_built_successor(r); - } - - void internal_delete_built_successor(successor_type &r) __TBB_override { - my_successors.internal_delete_built_successor(r); - } - - size_t successor_count() __TBB_override { - return my_successors.successor_count(); - } - - void copy_successors(successor_list_type &v) __TBB_override { - my_successors.copy_successors(v); - } - - typedef typename receiver::built_predecessors_type built_predecessors_type; - - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - void internal_add_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l(pred_mutex); - my_built_predecessors.add_edge(p); - } - - void internal_delete_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l(pred_mutex); - my_built_predecessors.delete_edge(p); - } - - size_t predecessor_count() __TBB_override { - spin_mutex::scoped_lock l(pred_mutex); - return my_built_predecessors.edge_count(); - } - - void copy_predecessors(predecessor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l(pred_mutex); - my_built_predecessors.copy_edges(v); - } - - void extract() __TBB_override { - my_built_predecessors.receiver_extract(*this); - my_successors.built_successors().sender_extract(*this); - } -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - //! build a task to run the successor if possible. Default is old behavior. - task *try_put_task(const T& t) __TBB_override { - task *new_task = my_successors.try_put_task(t); - if (!new_task) new_task = SUCCESSFULLY_ENQUEUED; - return new_task; - } - - graph& graph_reference() const __TBB_override { - return my_graph; - } - - void reset_receiver(reset_flags /*f*/) __TBB_override {} - - void reset_node(reset_flags f) __TBB_override { - if (f&rf_clear_edges) { - my_successors.clear(); -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - my_built_predecessors.clear(); -#endif - } - __TBB_ASSERT(!(f & rf_clear_edges) || my_successors.empty(), "Error resetting broadcast_node"); - } -}; // broadcast_node - -//! Forwards messages in arbitrary order -template > -class buffer_node : public graph_node, public internal::reservable_item_buffer, public receiver, public sender { -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - typedef buffer_node class_type; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename sender::successor_list_type successor_list_type; -#endif -protected: - typedef size_t size_type; - internal::round_robin_cache< T, null_rw_mutex > my_successors; - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - internal::edge_container my_built_predecessors; -#endif - - friend class internal::forward_task_bypass< buffer_node< T, A > >; - - enum op_type {reg_succ, rem_succ, req_item, res_item, rel_res, con_res, put_item, try_fwd_task -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - , add_blt_succ, del_blt_succ, - add_blt_pred, del_blt_pred, - blt_succ_cnt, blt_pred_cnt, - blt_succ_cpy, blt_pred_cpy // create vector copies of preds and succs -#endif - }; - - // implements the aggregator_operation concept - class buffer_operation : public internal::aggregated_operation< buffer_operation > { - public: - char type; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - task * ltask; - union { - input_type *elem; - successor_type *r; - predecessor_type *p; - size_t cnt_val; - successor_list_type *svec; - predecessor_list_type *pvec; - }; -#else - T *elem; - task * ltask; - successor_type *r; -#endif - buffer_operation(const T& e, op_type t) : type(char(t)) - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - , ltask(NULL), elem(const_cast(&e)) -#else - , elem(const_cast(&e)) , ltask(NULL) -#endif - {} - buffer_operation(op_type t) : type(char(t)), ltask(NULL) {} - }; - - bool forwarder_busy; - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - internal::aggregator< handler_type, buffer_operation> my_aggregator; - - virtual void handle_operations(buffer_operation *op_list) { - handle_operations_impl(op_list, this); - } - - template - void handle_operations_impl(buffer_operation *op_list, derived_type* derived) { - __TBB_ASSERT(static_cast(derived) == this, "'this' is not a base class for derived"); - - buffer_operation *tmp = NULL; - bool try_forwarding = false; - while (op_list) { - tmp = op_list; - op_list = op_list->next; - switch (tmp->type) { - case reg_succ: internal_reg_succ(tmp); try_forwarding = true; break; - case rem_succ: internal_rem_succ(tmp); break; - case req_item: internal_pop(tmp); break; - case res_item: internal_reserve(tmp); break; - case rel_res: internal_release(tmp); try_forwarding = true; break; - case con_res: internal_consume(tmp); try_forwarding = true; break; - case put_item: try_forwarding = internal_push(tmp); break; - case try_fwd_task: internal_forward_task(tmp); break; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - // edge recording - case add_blt_succ: internal_add_built_succ(tmp); break; - case del_blt_succ: internal_del_built_succ(tmp); break; - case add_blt_pred: internal_add_built_pred(tmp); break; - case del_blt_pred: internal_del_built_pred(tmp); break; - case blt_succ_cnt: internal_succ_cnt(tmp); break; - case blt_pred_cnt: internal_pred_cnt(tmp); break; - case blt_succ_cpy: internal_copy_succs(tmp); break; - case blt_pred_cpy: internal_copy_preds(tmp); break; -#endif - } - } - - derived->order(); - - if (try_forwarding && !forwarder_busy) { - if(internal::is_graph_active(this->my_graph)) { - forwarder_busy = true; - task *new_task = new(task::allocate_additional_child_of(*(this->my_graph.root_task()))) internal:: - forward_task_bypass - < buffer_node >(*this); - // tmp should point to the last item handled by the aggregator. This is the operation - // the handling thread enqueued. So modifying that record will be okay. - // workaround for icc bug - tbb::task *z = tmp->ltask; - graph &g = this->my_graph; - tmp->ltask = combine_tasks(g, z, new_task); // in case the op generated a task - } - } - } // handle_operations - - inline task *grab_forwarding_task( buffer_operation &op_data) { - return op_data.ltask; - } - - inline bool enqueue_forwarding_task(buffer_operation &op_data) { - task *ft = grab_forwarding_task(op_data); - if(ft) { - internal::spawn_in_graph_arena(graph_reference(), *ft); - return true; - } - return false; - } - - //! This is executed by an enqueued task, the "forwarder" - virtual task *forward_task() { - buffer_operation op_data(try_fwd_task); - task *last_task = NULL; - do { - op_data.status = internal::WAIT; - op_data.ltask = NULL; - my_aggregator.execute(&op_data); - - // workaround for icc bug - tbb::task *xtask = op_data.ltask; - graph& g = this->my_graph; - last_task = combine_tasks(g, last_task, xtask); - } while (op_data.status ==internal::SUCCEEDED); - return last_task; - } - - //! Register successor - virtual void internal_reg_succ(buffer_operation *op) { - my_successors.register_successor(*(op->r)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - //! Remove successor - virtual void internal_rem_succ(buffer_operation *op) { - my_successors.remove_successor(*(op->r)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename sender::built_successors_type built_successors_type; - - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - virtual void internal_add_built_succ(buffer_operation *op) { - my_successors.internal_add_built_successor(*(op->r)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_del_built_succ(buffer_operation *op) { - my_successors.internal_delete_built_successor(*(op->r)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - typedef typename receiver::built_predecessors_type built_predecessors_type; - - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - virtual void internal_add_built_pred(buffer_operation *op) { - my_built_predecessors.add_edge(*(op->p)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_del_built_pred(buffer_operation *op) { - my_built_predecessors.delete_edge(*(op->p)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_succ_cnt(buffer_operation *op) { - op->cnt_val = my_successors.successor_count(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_pred_cnt(buffer_operation *op) { - op->cnt_val = my_built_predecessors.edge_count(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_copy_succs(buffer_operation *op) { - my_successors.copy_successors(*(op->svec)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_copy_preds(buffer_operation *op) { - my_built_predecessors.copy_edges(*(op->pvec)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - -private: - void order() {} - - bool is_item_valid() { - return this->my_item_valid(this->my_tail - 1); - } - - void try_put_and_add_task(task*& last_task) { - task *new_task = my_successors.try_put_task(this->back()); - if (new_task) { - // workaround for icc bug - graph& g = this->my_graph; - last_task = combine_tasks(g, last_task, new_task); - this->destroy_back(); - } - } - -protected: - //! Tries to forward valid items to successors - virtual void internal_forward_task(buffer_operation *op) { - internal_forward_task_impl(op, this); - } - - template - void internal_forward_task_impl(buffer_operation *op, derived_type* derived) { - __TBB_ASSERT(static_cast(derived) == this, "'this' is not a base class for derived"); - - if (this->my_reserved || !derived->is_item_valid()) { - __TBB_store_with_release(op->status, internal::FAILED); - this->forwarder_busy = false; - return; - } - // Try forwarding, giving each successor a chance - task * last_task = NULL; - size_type counter = my_successors.size(); - for (; counter > 0 && derived->is_item_valid(); --counter) - derived->try_put_and_add_task(last_task); - - op->ltask = last_task; // return task - if (last_task && !counter) { - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - else { - __TBB_store_with_release(op->status, internal::FAILED); - forwarder_busy = false; - } - } - - virtual bool internal_push(buffer_operation *op) { - this->push_back(*(op->elem)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - return true; - } - - virtual void internal_pop(buffer_operation *op) { - if(this->pop_back(*(op->elem))) { - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - else { - __TBB_store_with_release(op->status, internal::FAILED); - } - } - - virtual void internal_reserve(buffer_operation *op) { - if(this->reserve_front(*(op->elem))) { - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - else { - __TBB_store_with_release(op->status, internal::FAILED); - } - } - - virtual void internal_consume(buffer_operation *op) { - this->consume_front(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - - virtual void internal_release(buffer_operation *op) { - this->release_front(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - -public: - //! Constructor - __TBB_NOINLINE_SYM explicit buffer_node( graph &g ) : graph_node(g), internal::reservable_item_buffer(), - forwarder_busy(false) { - my_successors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - tbb::internal::fgt_node( CODEPTR(), tbb::internal::FLOW_BUFFER_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - buffer_node(const node_set& nodes) : buffer_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - //! Copy constructor - __TBB_NOINLINE_SYM buffer_node( const buffer_node& src ) : graph_node(src.my_graph), - internal::reservable_item_buffer(), receiver(), sender() { - forwarder_busy = false; - my_successors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - tbb::internal::fgt_node( CODEPTR(), tbb::internal::FLOW_BUFFER_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - // - // message sender implementation - // - - //! Adds a new successor. - /** Adds successor r to the list of successors; may forward tasks. */ - bool register_successor( successor_type &r ) __TBB_override { - buffer_operation op_data(reg_succ); - op_data.r = &r; - my_aggregator.execute(&op_data); - (void)enqueue_forwarding_task(op_data); - return true; - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - void internal_add_built_successor( successor_type &r) __TBB_override { - buffer_operation op_data(add_blt_succ); - op_data.r = &r; - my_aggregator.execute(&op_data); - } - - void internal_delete_built_successor( successor_type &r) __TBB_override { - buffer_operation op_data(del_blt_succ); - op_data.r = &r; - my_aggregator.execute(&op_data); - } - - void internal_add_built_predecessor( predecessor_type &p) __TBB_override { - buffer_operation op_data(add_blt_pred); - op_data.p = &p; - my_aggregator.execute(&op_data); - } - - void internal_delete_built_predecessor( predecessor_type &p) __TBB_override { - buffer_operation op_data(del_blt_pred); - op_data.p = &p; - my_aggregator.execute(&op_data); - } - - size_t predecessor_count() __TBB_override { - buffer_operation op_data(blt_pred_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - size_t successor_count() __TBB_override { - buffer_operation op_data(blt_succ_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_predecessors( predecessor_list_type &v ) __TBB_override { - buffer_operation op_data(blt_pred_cpy); - op_data.pvec = &v; - my_aggregator.execute(&op_data); - } - - void copy_successors( successor_list_type &v ) __TBB_override { - buffer_operation op_data(blt_succ_cpy); - op_data.svec = &v; - my_aggregator.execute(&op_data); - } - -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - - //! Removes a successor. - /** Removes successor r from the list of successors. - It also calls r.remove_predecessor(*this) to remove this node as a predecessor. */ - bool remove_successor( successor_type &r ) __TBB_override { - r.remove_predecessor(*this); - buffer_operation op_data(rem_succ); - op_data.r = &r; - my_aggregator.execute(&op_data); - // even though this operation does not cause a forward, if we are the handler, and - // a forward is scheduled, we may be the first to reach this point after the aggregator, - // and so should check for the task. - (void)enqueue_forwarding_task(op_data); - return true; - } - - //! Request an item from the buffer_node - /** true = v contains the returned item
- false = no item has been returned */ - bool try_get( T &v ) __TBB_override { - buffer_operation op_data(req_item); - op_data.elem = &v; - my_aggregator.execute(&op_data); - (void)enqueue_forwarding_task(op_data); - return (op_data.status==internal::SUCCEEDED); - } - - //! Reserves an item. - /** false = no item can be reserved
- true = an item is reserved */ - bool try_reserve( T &v ) __TBB_override { - buffer_operation op_data(res_item); - op_data.elem = &v; - my_aggregator.execute(&op_data); - (void)enqueue_forwarding_task(op_data); - return (op_data.status==internal::SUCCEEDED); - } - - //! Release a reserved item. - /** true = item has been released and so remains in sender */ - bool try_release() __TBB_override { - buffer_operation op_data(rel_res); - my_aggregator.execute(&op_data); - (void)enqueue_forwarding_task(op_data); - return true; - } - - //! Consumes a reserved item. - /** true = item is removed from sender and reservation removed */ - bool try_consume() __TBB_override { - buffer_operation op_data(con_res); - my_aggregator.execute(&op_data); - (void)enqueue_forwarding_task(op_data); - return true; - } - -protected: - - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - //! receive an item, return a task *if possible - task *try_put_task(const T &t) __TBB_override { - buffer_operation op_data(t, put_item); - my_aggregator.execute(&op_data); - task *ft = grab_forwarding_task(op_data); - // sequencer_nodes can return failure (if an item has been previously inserted) - // We have to spawn the returned task if our own operation fails. - - if(ft && op_data.status ==internal::FAILED) { - // we haven't succeeded queueing the item, but for some reason the - // call returned a task (if another request resulted in a successful - // forward this could happen.) Queue the task and reset the pointer. - internal::spawn_in_graph_arena(graph_reference(), *ft); ft = NULL; - } - else if(!ft && op_data.status ==internal::SUCCEEDED) { - ft = SUCCESSFULLY_ENQUEUED; - } - return ft; - } - - graph& graph_reference() const __TBB_override { - return my_graph; - } - - void reset_receiver(reset_flags /*f*/) __TBB_override { } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION -public: - void extract() __TBB_override { - my_built_predecessors.receiver_extract(*this); - my_successors.built_successors().sender_extract(*this); - } -#endif - -protected: - void reset_node( reset_flags f) __TBB_override { - internal::reservable_item_buffer::reset(); - // TODO: just clear structures - if (f&rf_clear_edges) { - my_successors.clear(); -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - my_built_predecessors.clear(); -#endif - } - forwarder_busy = false; - } -}; // buffer_node - -//! Forwards messages in FIFO order -template > -class queue_node : public buffer_node { -protected: - typedef buffer_node base_type; - typedef typename base_type::size_type size_type; - typedef typename base_type::buffer_operation queue_operation; - typedef queue_node class_type; - -private: - template friend class buffer_node; - - bool is_item_valid() { - return this->my_item_valid(this->my_head); - } - - void try_put_and_add_task(task*& last_task) { - task *new_task = this->my_successors.try_put_task(this->front()); - if (new_task) { - // workaround for icc bug - graph& graph_ref = this->graph_reference(); - last_task = combine_tasks(graph_ref, last_task, new_task); - this->destroy_front(); - } - } - -protected: - void internal_forward_task(queue_operation *op) __TBB_override { - this->internal_forward_task_impl(op, this); - } - - void internal_pop(queue_operation *op) __TBB_override { - if ( this->my_reserved || !this->my_item_valid(this->my_head)){ - __TBB_store_with_release(op->status, internal::FAILED); - } - else { - this->pop_front(*(op->elem)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - } - void internal_reserve(queue_operation *op) __TBB_override { - if (this->my_reserved || !this->my_item_valid(this->my_head)) { - __TBB_store_with_release(op->status, internal::FAILED); - } - else { - this->reserve_front(*(op->elem)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - } - void internal_consume(queue_operation *op) __TBB_override { - this->consume_front(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - } - -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - - //! Constructor - __TBB_NOINLINE_SYM explicit queue_node( graph &g ) : base_type(g) { - tbb::internal::fgt_node( CODEPTR(), tbb::internal::FLOW_QUEUE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - queue_node( const node_set& nodes) : queue_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - //! Copy constructor - __TBB_NOINLINE_SYM queue_node( const queue_node& src) : base_type(src) { - tbb::internal::fgt_node( CODEPTR(), tbb::internal::FLOW_QUEUE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -protected: - void reset_node( reset_flags f) __TBB_override { - base_type::reset_node(f); - } -}; // queue_node - -//! Forwards messages in sequence order -template< typename T, typename A=cache_aligned_allocator > -class sequencer_node : public queue_node { - internal::function_body< T, size_t > *my_sequencer; - // my_sequencer should be a benign function and must be callable - // from a parallel context. Does this mean it needn't be reset? -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - - //! Constructor - template< typename Sequencer > - __TBB_NOINLINE_SYM sequencer_node( graph &g, const Sequencer& s ) : queue_node(g), - my_sequencer(new internal::function_body_leaf< T, size_t, Sequencer>(s) ) { - tbb::internal::fgt_node( CODEPTR(), tbb::internal::FLOW_SEQUENCER_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - sequencer_node( const node_set& nodes, const Sequencer& s) - : sequencer_node(nodes.graph_reference(), s) { - make_edges_in_order(nodes, *this); - } -#endif - - //! Copy constructor - __TBB_NOINLINE_SYM sequencer_node( const sequencer_node& src ) : queue_node(src), - my_sequencer( src.my_sequencer->clone() ) { - tbb::internal::fgt_node( CODEPTR(), tbb::internal::FLOW_SEQUENCER_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - - //! Destructor - ~sequencer_node() { delete my_sequencer; } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -protected: - typedef typename buffer_node::size_type size_type; - typedef typename buffer_node::buffer_operation sequencer_operation; - -private: - bool internal_push(sequencer_operation *op) __TBB_override { - size_type tag = (*my_sequencer)(*(op->elem)); -#if !TBB_DEPRECATED_SEQUENCER_DUPLICATES - if (tag < this->my_head) { - // have already emitted a message with this tag - __TBB_store_with_release(op->status, internal::FAILED); - return false; - } -#endif - // cannot modify this->my_tail now; the buffer would be inconsistent. - size_t new_tail = (tag+1 > this->my_tail) ? tag+1 : this->my_tail; - - if (this->size(new_tail) > this->capacity()) { - this->grow_my_array(this->size(new_tail)); - } - this->my_tail = new_tail; - - const internal::op_stat res = this->place_item(tag, *(op->elem)) ? internal::SUCCEEDED : internal::FAILED; - __TBB_store_with_release(op->status, res); - return res ==internal::SUCCEEDED; - } -}; // sequencer_node - -//! Forwards messages in priority order -template< typename T, typename Compare = std::less, typename A=cache_aligned_allocator > -class priority_queue_node : public buffer_node { -public: - typedef T input_type; - typedef T output_type; - typedef buffer_node base_type; - typedef priority_queue_node class_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - - //! Constructor - __TBB_NOINLINE_SYM explicit priority_queue_node( graph &g, const Compare& comp = Compare() ) - : buffer_node(g), compare(comp), mark(0) { - tbb::internal::fgt_node( CODEPTR(), tbb::internal::FLOW_PRIORITY_QUEUE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - priority_queue_node(const node_set& nodes, const Compare& comp = Compare()) - : priority_queue_node(nodes.graph_reference(), comp) { - make_edges_in_order(nodes, *this); - } -#endif - - //! Copy constructor - __TBB_NOINLINE_SYM priority_queue_node( const priority_queue_node &src ) : buffer_node(src), mark(0) { - tbb::internal::fgt_node( CODEPTR(), tbb::internal::FLOW_PRIORITY_QUEUE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -protected: - - void reset_node( reset_flags f) __TBB_override { - mark = 0; - base_type::reset_node(f); - } - - typedef typename buffer_node::size_type size_type; - typedef typename buffer_node::item_type item_type; - typedef typename buffer_node::buffer_operation prio_operation; - - //! Tries to forward valid items to successors - void internal_forward_task(prio_operation *op) __TBB_override { - this->internal_forward_task_impl(op, this); - } - - void handle_operations(prio_operation *op_list) __TBB_override { - this->handle_operations_impl(op_list, this); - } - - bool internal_push(prio_operation *op) __TBB_override { - prio_push(*(op->elem)); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - return true; - } - - void internal_pop(prio_operation *op) __TBB_override { - // if empty or already reserved, don't pop - if ( this->my_reserved == true || this->my_tail == 0 ) { - __TBB_store_with_release(op->status, internal::FAILED); - return; - } - - *(op->elem) = prio(); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - prio_pop(); - - } - - // pops the highest-priority item, saves copy - void internal_reserve(prio_operation *op) __TBB_override { - if (this->my_reserved == true || this->my_tail == 0) { - __TBB_store_with_release(op->status, internal::FAILED); - return; - } - this->my_reserved = true; - *(op->elem) = prio(); - reserved_item = *(op->elem); - __TBB_store_with_release(op->status, internal::SUCCEEDED); - prio_pop(); - } - - void internal_consume(prio_operation *op) __TBB_override { - __TBB_store_with_release(op->status, internal::SUCCEEDED); - this->my_reserved = false; - reserved_item = input_type(); - } - - void internal_release(prio_operation *op) __TBB_override { - __TBB_store_with_release(op->status, internal::SUCCEEDED); - prio_push(reserved_item); - this->my_reserved = false; - reserved_item = input_type(); - } - -private: - template friend class buffer_node; - - void order() { - if (mark < this->my_tail) heapify(); - __TBB_ASSERT(mark == this->my_tail, "mark unequal after heapify"); - } - - bool is_item_valid() { - return this->my_tail > 0; - } - - void try_put_and_add_task(task*& last_task) { - task * new_task = this->my_successors.try_put_task(this->prio()); - if (new_task) { - // workaround for icc bug - graph& graph_ref = this->graph_reference(); - last_task = combine_tasks(graph_ref, last_task, new_task); - prio_pop(); - } - } - -private: - Compare compare; - size_type mark; - - input_type reserved_item; - - // in case a reheap has not been done after a push, check if the mark item is higher than the 0'th item - bool prio_use_tail() { - __TBB_ASSERT(mark <= this->my_tail, "mark outside bounds before test"); - return mark < this->my_tail && compare(this->get_my_item(0), this->get_my_item(this->my_tail - 1)); - } - - // prio_push: checks that the item will fit, expand array if necessary, put at end - void prio_push(const T &src) { - if ( this->my_tail >= this->my_array_size ) - this->grow_my_array( this->my_tail + 1 ); - (void) this->place_item(this->my_tail, src); - ++(this->my_tail); - __TBB_ASSERT(mark < this->my_tail, "mark outside bounds after push"); - } - - // prio_pop: deletes highest priority item from the array, and if it is item - // 0, move last item to 0 and reheap. If end of array, just destroy and decrement tail - // and mark. Assumes the array has already been tested for emptiness; no failure. - void prio_pop() { - if (prio_use_tail()) { - // there are newly pushed elements; last one higher than top - // copy the data - this->destroy_item(this->my_tail-1); - --(this->my_tail); - __TBB_ASSERT(mark <= this->my_tail, "mark outside bounds after pop"); - return; - } - this->destroy_item(0); - if(this->my_tail > 1) { - // push the last element down heap - __TBB_ASSERT(this->my_item_valid(this->my_tail - 1), NULL); - this->move_item(0,this->my_tail - 1); - } - --(this->my_tail); - if(mark > this->my_tail) --mark; - if (this->my_tail > 1) // don't reheap for heap of size 1 - reheap(); - __TBB_ASSERT(mark <= this->my_tail, "mark outside bounds after pop"); - } - - const T& prio() { - return this->get_my_item(prio_use_tail() ? this->my_tail-1 : 0); - } - - // turn array into heap - void heapify() { - if(this->my_tail == 0) { - mark = 0; - return; - } - if (!mark) mark = 1; - for (; markmy_tail; ++mark) { // for each unheaped element - size_type cur_pos = mark; - input_type to_place; - this->fetch_item(mark,to_place); - do { // push to_place up the heap - size_type parent = (cur_pos-1)>>1; - if (!compare(this->get_my_item(parent), to_place)) - break; - this->move_item(cur_pos, parent); - cur_pos = parent; - } while( cur_pos ); - (void) this->place_item(cur_pos, to_place); - } - } - - // otherwise heapified array with new root element; rearrange to heap - void reheap() { - size_type cur_pos=0, child=1; - while (child < mark) { - size_type target = child; - if (child+1get_my_item(child), - this->get_my_item(child+1))) - ++target; - // target now has the higher priority child - if (compare(this->get_my_item(target), - this->get_my_item(cur_pos))) - break; - // swap - this->swap_items(cur_pos, target); - cur_pos = target; - child = (cur_pos<<1)+1; - } - } -}; // priority_queue_node - -} // interfaceX - -namespace interface11 { - -//! Forwards messages only if the threshold has not been reached -/** This node forwards items until its threshold is reached. - It contains no buffering. If the downstream node rejects, the - message is dropped. */ -template< typename T, typename DecrementType=continue_msg > -class limiter_node : public graph_node, public receiver< T >, public sender< T > { -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename receiver::built_predecessors_type built_predecessors_type; - typedef typename sender::built_successors_type built_successors_type; - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - //TODO: There is a lack of predefined types for its controlling "decrementer" port. It should be fixed later. - -private: - size_t my_threshold; - size_t my_count; //number of successful puts - size_t my_tries; //number of active put attempts - internal::reservable_predecessor_cache< T, spin_mutex > my_predecessors; - spin_mutex my_mutex; - internal::broadcast_cache< T > my_successors; - __TBB_DEPRECATED_LIMITER_EXPR( int init_decrement_predecessors; ) - - friend class internal::forward_task_bypass< limiter_node >; - - // Let decrementer call decrement_counter() - friend class internal::decrementer< limiter_node, DecrementType >; - - bool check_conditions() { // always called under lock - return ( my_count + my_tries < my_threshold && !my_predecessors.empty() && !my_successors.empty() ); - } - - // only returns a valid task pointer or NULL, never SUCCESSFULLY_ENQUEUED - task *forward_task() { - input_type v; - task *rval = NULL; - bool reserved = false; - { - spin_mutex::scoped_lock lock(my_mutex); - if ( check_conditions() ) - ++my_tries; - else - return NULL; - } - - //SUCCESS - // if we can reserve and can put, we consume the reservation - // we increment the count and decrement the tries - if ( (my_predecessors.try_reserve(v)) == true ){ - reserved=true; - if ( (rval = my_successors.try_put_task(v)) != NULL ){ - { - spin_mutex::scoped_lock lock(my_mutex); - ++my_count; - --my_tries; - my_predecessors.try_consume(); - if ( check_conditions() ) { - if ( internal::is_graph_active(this->my_graph) ) { - task *rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); - internal::spawn_in_graph_arena(graph_reference(), *rtask); - } - } - } - return rval; - } - } - //FAILURE - //if we can't reserve, we decrement the tries - //if we can reserve but can't put, we decrement the tries and release the reservation - { - spin_mutex::scoped_lock lock(my_mutex); - --my_tries; - if (reserved) my_predecessors.try_release(); - if ( check_conditions() ) { - if ( internal::is_graph_active(this->my_graph) ) { - task *rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); - __TBB_ASSERT(!rval, "Have two tasks to handle"); - return rtask; - } - } - return rval; - } - } - - void forward() { - __TBB_ASSERT(false, "Should never be called"); - return; - } - - task* decrement_counter( long long delta ) { - { - spin_mutex::scoped_lock lock(my_mutex); - if( delta > 0 && size_t(delta) > my_count ) - my_count = 0; - else if( delta < 0 && size_t(delta) > my_threshold - my_count ) - my_count = my_threshold; - else - my_count -= size_t(delta); // absolute value of delta is sufficiently small - } - return forward_task(); - } - - void initialize() { - my_predecessors.set_owner(this); - my_successors.set_owner(this); - decrement.set_owner(this); - tbb::internal::fgt_node( - CODEPTR(), tbb::internal::FLOW_LIMITER_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(&decrement), - static_cast *>(this) - ); - } -public: - //! The internal receiver< DecrementType > that decrements the count - internal::decrementer< limiter_node, DecrementType > decrement; - -#if TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR - __TBB_STATIC_ASSERT( (tbb::internal::is_same_type::value), - "Deprecated interface of the limiter node can be used only in conjunction " - "with continue_msg as the type of DecrementType template parameter." ); -#endif // Check for incompatible interface - - //! Constructor - limiter_node(graph &g, - __TBB_DEPRECATED_LIMITER_ARG2(size_t threshold, int num_decrement_predecessors=0)) - : graph_node(g), my_threshold(threshold), my_count(0), - __TBB_DEPRECATED_LIMITER_ARG4( - my_tries(0), decrement(), - init_decrement_predecessors(num_decrement_predecessors), - decrement(num_decrement_predecessors)) { - initialize(); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - limiter_node(const node_set& nodes, size_t threshold) - : limiter_node(nodes.graph_reference(), threshold) { - make_edges_in_order(nodes, *this); - } -#endif - - //! Copy constructor - limiter_node( const limiter_node& src ) : - graph_node(src.my_graph), receiver(), sender(), - my_threshold(src.my_threshold), my_count(0), - __TBB_DEPRECATED_LIMITER_ARG4( - my_tries(0), decrement(), - init_decrement_predecessors(src.init_decrement_predecessors), - decrement(src.init_decrement_predecessors)) { - initialize(); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - //! Replace the current successor with this new successor - bool register_successor( successor_type &r ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - bool was_empty = my_successors.empty(); - my_successors.register_successor(r); - //spawn a forward task if this is the only successor - if ( was_empty && !my_predecessors.empty() && my_count + my_tries < my_threshold ) { - if ( internal::is_graph_active(this->my_graph) ) { - task* task = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass < limiter_node >( *this ); - internal::spawn_in_graph_arena(graph_reference(), *task); - } - } - return true; - } - - //! Removes a successor from this node - /** r.remove_predecessor(*this) is also called. */ - bool remove_successor( successor_type &r ) __TBB_override { - r.remove_predecessor(*this); - my_successors.remove_successor(r); - return true; - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - built_predecessors_type &built_predecessors() __TBB_override { return my_predecessors.built_predecessors(); } - - void internal_add_built_successor(successor_type &src) __TBB_override { - my_successors.internal_add_built_successor(src); - } - - void internal_delete_built_successor(successor_type &src) __TBB_override { - my_successors.internal_delete_built_successor(src); - } - - size_t successor_count() __TBB_override { return my_successors.successor_count(); } - - void copy_successors(successor_list_type &v) __TBB_override { - my_successors.copy_successors(v); - } - - void internal_add_built_predecessor(predecessor_type &src) __TBB_override { - my_predecessors.internal_add_built_predecessor(src); - } - - void internal_delete_built_predecessor(predecessor_type &src) __TBB_override { - my_predecessors.internal_delete_built_predecessor(src); - } - - size_t predecessor_count() __TBB_override { return my_predecessors.predecessor_count(); } - - void copy_predecessors(predecessor_list_type &v) __TBB_override { - my_predecessors.copy_predecessors(v); - } - - void extract() __TBB_override { - my_count = 0; - my_successors.built_successors().sender_extract(*this); - my_predecessors.built_predecessors().receiver_extract(*this); - decrement.built_predecessors().receiver_extract(decrement); - } -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - - //! Adds src to the list of cached predecessors. - bool register_predecessor( predecessor_type &src ) __TBB_override { - spin_mutex::scoped_lock lock(my_mutex); - my_predecessors.add( src ); - if ( my_count + my_tries < my_threshold && !my_successors.empty() && internal::is_graph_active(this->my_graph) ) { - task* task = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass < limiter_node >( *this ); - internal::spawn_in_graph_arena(graph_reference(), *task); - } - return true; - } - - //! Removes src from the list of cached predecessors. - bool remove_predecessor( predecessor_type &src ) __TBB_override { - my_predecessors.remove( src ); - return true; - } - -protected: - - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - //! Puts an item to this receiver - task *try_put_task( const T &t ) __TBB_override { - { - spin_mutex::scoped_lock lock(my_mutex); - if ( my_count + my_tries >= my_threshold ) - return NULL; - else - ++my_tries; - } - - task * rtask = my_successors.try_put_task(t); - - if ( !rtask ) { // try_put_task failed. - spin_mutex::scoped_lock lock(my_mutex); - --my_tries; - if (check_conditions() && internal::is_graph_active(this->my_graph)) { - rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); - } - } - else { - spin_mutex::scoped_lock lock(my_mutex); - ++my_count; - --my_tries; - } - return rtask; - } - - graph& graph_reference() const __TBB_override { return my_graph; } - - void reset_receiver(reset_flags /*f*/) __TBB_override { - __TBB_ASSERT(false,NULL); // should never be called - } - - void reset_node( reset_flags f) __TBB_override { - my_count = 0; - if(f & rf_clear_edges) { - my_predecessors.clear(); - my_successors.clear(); - } - else - { - my_predecessors.reset( ); - } - decrement.reset_receiver(f); - } -}; // limiter_node - -#include "internal/_flow_graph_join_impl.h" - -using internal::reserving_port; -using internal::queueing_port; -using internal::key_matching_port; -using internal::input_port; -using internal::tag_value; - -template class join_node; - -template -class join_node: public internal::unfolded_join_node::value, reserving_port, OutputTuple, reserving> { -private: - static const int N = tbb::flow::tuple_size::value; - typedef typename internal::unfolded_join_node unfolded_type; -public: - typedef OutputTuple output_type; - typedef typename unfolded_type::input_ports_type input_ports_type; - __TBB_NOINLINE_SYM explicit join_node(graph &g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_JOIN_NODE_RESERVING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - __TBB_NOINLINE_SYM join_node(const node_set& nodes, reserving = reserving()) : join_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - __TBB_NOINLINE_SYM join_node(const join_node &other) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_JOIN_NODE_RESERVING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -}; - -template -class join_node: public internal::unfolded_join_node::value, queueing_port, OutputTuple, queueing> { -private: - static const int N = tbb::flow::tuple_size::value; - typedef typename internal::unfolded_join_node unfolded_type; -public: - typedef OutputTuple output_type; - typedef typename unfolded_type::input_ports_type input_ports_type; - __TBB_NOINLINE_SYM explicit join_node(graph &g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_JOIN_NODE_QUEUEING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - __TBB_NOINLINE_SYM join_node(const node_set& nodes, queueing = queueing()) : join_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - __TBB_NOINLINE_SYM join_node(const join_node &other) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_JOIN_NODE_QUEUEING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -}; - -// template for key_matching join_node -// tag_matching join_node is a specialization of key_matching, and is source-compatible. -template -class join_node > : public internal::unfolded_join_node::value, - key_matching_port, OutputTuple, key_matching > { -private: - static const int N = tbb::flow::tuple_size::value; - typedef typename internal::unfolded_join_node > unfolded_type; -public: - typedef OutputTuple output_type; - typedef typename unfolded_type::input_ports_type input_ports_type; - -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - join_node(graph &g) : unfolded_type(g) {} - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - join_node(const node_set& nodes, key_matching = key_matching()) - : join_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - - template - __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1) : unfolded_type(g, b0, b1) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - template - __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2) : unfolded_type(g, b0, b1, b2) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - template - __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3) : unfolded_type(g, b0, b1, b2, b3) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - template - __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4) : - unfolded_type(g, b0, b1, b2, b3, b4) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#if __TBB_VARIADIC_MAX >= 6 - template - __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5) : - unfolded_type(g, b0, b1, b2, b3, b4, b5) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#endif -#if __TBB_VARIADIC_MAX >= 7 - template - __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6) : - unfolded_type(g, b0, b1, b2, b3, b4, b5, b6) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#endif -#if __TBB_VARIADIC_MAX >= 8 - template - __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6, - __TBB_B7 b7) : unfolded_type(g, b0, b1, b2, b3, b4, b5, b6, b7) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#endif -#if __TBB_VARIADIC_MAX >= 9 - template - __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6, - __TBB_B7 b7, __TBB_B8 b8) : unfolded_type(g, b0, b1, b2, b3, b4, b5, b6, b7, b8) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#endif -#if __TBB_VARIADIC_MAX >= 10 - template - __TBB_NOINLINE_SYM join_node(graph &g, __TBB_B0 b0, __TBB_B1 b1, __TBB_B2 b2, __TBB_B3 b3, __TBB_B4 b4, __TBB_B5 b5, __TBB_B6 b6, - __TBB_B7 b7, __TBB_B8 b8, __TBB_B9 b9) : unfolded_type(g, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } -#endif - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - __TBB_NOINLINE_SYM join_node(const node_set& nodes, Bodies... bodies) - : join_node(nodes.graph_reference(), bodies...) { - make_edges_in_order(nodes, *this); - } -#endif - - __TBB_NOINLINE_SYM join_node(const join_node &other) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_JOIN_NODE_TAG_MATCHING, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -}; - -// indexer node -#include "internal/_flow_graph_indexer_impl.h" - -// TODO: Implement interface with variadic template or tuple -template class indexer_node; - -//indexer node specializations -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 1; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - // Copy constructor - __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; - -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 2; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - // Copy constructor - __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; - -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 3; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - // Copy constructor - __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; - -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 4; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - // Copy constructor - __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; - -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 5; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - // Copy constructor - __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; - -#if __TBB_VARIADIC_MAX >= 6 -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 6; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - // Copy constructor - __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; -#endif //variadic max 6 - -#if __TBB_VARIADIC_MAX >= 7 -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 7; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - // Copy constructor - __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; -#endif //variadic max 7 - -#if __TBB_VARIADIC_MAX >= 8 -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 8; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - // Copy constructor - indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; -#endif //variadic max 8 - -#if __TBB_VARIADIC_MAX >= 9 -template -class indexer_node : public internal::unfolded_indexer_node > { -private: - static const int N = 9; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - // Copy constructor - __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; -#endif //variadic max 9 - -#if __TBB_VARIADIC_MAX >= 10 -template -class indexer_node/*default*/ : public internal::unfolded_indexer_node > { -private: - static const int N = 10; -public: - typedef tuple InputTuple; - typedef typename internal::tagged_msg output_type; - typedef typename internal::unfolded_indexer_node unfolded_type; - __TBB_NOINLINE_SYM indexer_node(graph& g) : unfolded_type(g) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - indexer_node(const node_set& nodes) : indexer_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - // Copy constructor - __TBB_NOINLINE_SYM indexer_node( const indexer_node& other ) : unfolded_type(other) { - tbb::internal::fgt_multiinput_node( CODEPTR(), tbb::internal::FLOW_INDEXER_NODE, &this->my_graph, - this->input_ports(), static_cast< sender< output_type > *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif -}; -#endif //variadic max 10 - -#if __TBB_PREVIEW_ASYNC_MSG -inline void internal_make_edge( internal::untyped_sender &p, internal::untyped_receiver &s ) { -#else -template< typename T > -inline void internal_make_edge( sender &p, receiver &s ) { -#endif -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - s.internal_add_built_predecessor(p); - p.internal_add_built_successor(s); -#endif - p.register_successor( s ); - tbb::internal::fgt_make_edge( &p, &s ); -} - -//! Makes an edge between a single predecessor and a single successor -template< typename T > -inline void make_edge( sender &p, receiver &s ) { - internal_make_edge( p, s ); -} - -#if __TBB_PREVIEW_ASYNC_MSG -template< typename TS, typename TR, - typename = typename tbb::internal::enable_if::value - || tbb::internal::is_same_type::value>::type> -inline void make_edge( TS &p, TR &s ) { - internal_make_edge( p, s ); -} - -template< typename T > -inline void make_edge( sender &p, receiver &s ) { - internal_make_edge( p, s ); -} - -template< typename T > -inline void make_edge( sender &p, receiver &s ) { - internal_make_edge( p, s ); -} - -#endif // __TBB_PREVIEW_ASYNC_MSG - -#if __TBB_FLOW_GRAPH_CPP11_FEATURES -//Makes an edge from port 0 of a multi-output predecessor to port 0 of a multi-input successor. -template< typename T, typename V, - typename = typename T::output_ports_type, typename = typename V::input_ports_type > -inline void make_edge( T& output, V& input) { - make_edge(get<0>(output.output_ports()), get<0>(input.input_ports())); -} - -//Makes an edge from port 0 of a multi-output predecessor to a receiver. -template< typename T, typename R, - typename = typename T::output_ports_type > -inline void make_edge( T& output, receiver& input) { - make_edge(get<0>(output.output_ports()), input); -} - -//Makes an edge from a sender to port 0 of a multi-input successor. -template< typename S, typename V, - typename = typename V::input_ports_type > -inline void make_edge( sender& output, V& input) { - make_edge(output, get<0>(input.input_ports())); -} -#endif - -#if __TBB_PREVIEW_ASYNC_MSG -inline void internal_remove_edge( internal::untyped_sender &p, internal::untyped_receiver &s ) { -#else -template< typename T > -inline void internal_remove_edge( sender &p, receiver &s ) { -#endif - p.remove_successor( s ); -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - // TODO: should we try to remove p from the predecessor list of s, in case the edge is reversed? - p.internal_delete_built_successor(s); - s.internal_delete_built_predecessor(p); -#endif - tbb::internal::fgt_remove_edge( &p, &s ); -} - -//! Removes an edge between a single predecessor and a single successor -template< typename T > -inline void remove_edge( sender &p, receiver &s ) { - internal_remove_edge( p, s ); -} - -#if __TBB_PREVIEW_ASYNC_MSG -template< typename TS, typename TR, - typename = typename tbb::internal::enable_if::value - || tbb::internal::is_same_type::value>::type> -inline void remove_edge( TS &p, TR &s ) { - internal_remove_edge( p, s ); -} - -template< typename T > -inline void remove_edge( sender &p, receiver &s ) { - internal_remove_edge( p, s ); -} - -template< typename T > -inline void remove_edge( sender &p, receiver &s ) { - internal_remove_edge( p, s ); -} -#endif // __TBB_PREVIEW_ASYNC_MSG - -#if __TBB_FLOW_GRAPH_CPP11_FEATURES -//Removes an edge between port 0 of a multi-output predecessor and port 0 of a multi-input successor. -template< typename T, typename V, - typename = typename T::output_ports_type, typename = typename V::input_ports_type > -inline void remove_edge( T& output, V& input) { - remove_edge(get<0>(output.output_ports()), get<0>(input.input_ports())); -} - -//Removes an edge between port 0 of a multi-output predecessor and a receiver. -template< typename T, typename R, - typename = typename T::output_ports_type > -inline void remove_edge( T& output, receiver& input) { - remove_edge(get<0>(output.output_ports()), input); -} -//Removes an edge between a sender and port 0 of a multi-input successor. -template< typename S, typename V, - typename = typename V::input_ports_type > -inline void remove_edge( sender& output, V& input) { - remove_edge(output, get<0>(input.input_ports())); -} -#endif - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION -template -template< typename S > -void internal::edge_container::sender_extract( S &s ) { - edge_list_type e = built_edges; - for ( typename edge_list_type::iterator i = e.begin(); i != e.end(); ++i ) { - remove_edge(s, **i); - } -} - -template -template< typename R > -void internal::edge_container::receiver_extract( R &r ) { - edge_list_type e = built_edges; - for ( typename edge_list_type::iterator i = e.begin(); i != e.end(); ++i ) { - remove_edge(**i, r); - } -} -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - -//! Returns a copy of the body from a function or continue node -template< typename Body, typename Node > -Body copy_body( Node &n ) { - return n.template copy_function_object(); -} - -#if __TBB_FLOW_GRAPH_CPP11_FEATURES - -//composite_node -template< typename InputTuple, typename OutputTuple > class composite_node; - -template< typename... InputTypes, typename... OutputTypes> -class composite_node , tbb::flow::tuple > : public graph_node{ - -public: - typedef tbb::flow::tuple< receiver&... > input_ports_type; - typedef tbb::flow::tuple< sender&... > output_ports_type; - -private: - std::unique_ptr my_input_ports; - std::unique_ptr my_output_ports; - - static const size_t NUM_INPUTS = sizeof...(InputTypes); - static const size_t NUM_OUTPUTS = sizeof...(OutputTypes); - -protected: - void reset_node(reset_flags) __TBB_override {} - -public: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - composite_node( graph &g, const char *type_name = "composite_node" ) : graph_node(g) { - tbb::internal::fgt_multiinput_multioutput_node( CODEPTR(), tbb::internal::FLOW_COMPOSITE_NODE, this, &this->my_graph ); - tbb::internal::fgt_multiinput_multioutput_node_desc( this, type_name ); - } -#else - composite_node( graph &g ) : graph_node(g) { - tbb::internal::fgt_multiinput_multioutput_node( CODEPTR(), tbb::internal::FLOW_COMPOSITE_NODE, this, &this->my_graph ); - } -#endif - - template - void set_external_ports(T1&& input_ports_tuple, T2&& output_ports_tuple) { - __TBB_STATIC_ASSERT(NUM_INPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of input ports"); - __TBB_STATIC_ASSERT(NUM_OUTPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of output ports"); - my_input_ports = tbb::internal::make_unique(std::forward(input_ports_tuple)); - my_output_ports = tbb::internal::make_unique(std::forward(output_ports_tuple)); - - tbb::internal::fgt_internal_input_alias_helper::alias_port( this, input_ports_tuple); - tbb::internal::fgt_internal_output_alias_helper::alias_port( this, output_ports_tuple); - } - - template< typename... NodeTypes > - void add_visible_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, true, n...); } - - template< typename... NodeTypes > - void add_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, false, n...); } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multiinput_multioutput_node_desc( this, name ); - } -#endif - - input_ports_type& input_ports() { - __TBB_ASSERT(my_input_ports, "input ports not set, call set_external_ports to set input ports"); - return *my_input_ports; - } - - output_ports_type& output_ports() { - __TBB_ASSERT(my_output_ports, "output ports not set, call set_external_ports to set output ports"); - return *my_output_ports; - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - void extract() __TBB_override { - __TBB_ASSERT(false, "Current composite_node implementation does not support extract"); - } -#endif -}; // class composite_node - -//composite_node with only input ports -template< typename... InputTypes> -class composite_node , tbb::flow::tuple<> > : public graph_node { -public: - typedef tbb::flow::tuple< receiver&... > input_ports_type; - -private: - std::unique_ptr my_input_ports; - static const size_t NUM_INPUTS = sizeof...(InputTypes); - -protected: - void reset_node(reset_flags) __TBB_override {} - -public: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - composite_node( graph &g, const char *type_name = "composite_node") : graph_node(g) { - tbb::internal::fgt_composite( CODEPTR(), this, &g ); - tbb::internal::fgt_multiinput_multioutput_node_desc( this, type_name ); - } -#else - composite_node( graph &g ) : graph_node(g) { - tbb::internal::fgt_composite( CODEPTR(), this, &g ); - } -#endif - - template - void set_external_ports(T&& input_ports_tuple) { - __TBB_STATIC_ASSERT(NUM_INPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of input ports"); - - my_input_ports = tbb::internal::make_unique(std::forward(input_ports_tuple)); - - tbb::internal::fgt_internal_input_alias_helper::alias_port( this, std::forward(input_ports_tuple)); - } - - template< typename... NodeTypes > - void add_visible_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, true, n...); } - - template< typename... NodeTypes > - void add_nodes( const NodeTypes&... n) { internal::add_nodes_impl(this, false, n...); } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multiinput_multioutput_node_desc( this, name ); - } -#endif - - input_ports_type& input_ports() { - __TBB_ASSERT(my_input_ports, "input ports not set, call set_external_ports to set input ports"); - return *my_input_ports; - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - void extract() __TBB_override { - __TBB_ASSERT(false, "Current composite_node implementation does not support extract"); - } -#endif - -}; // class composite_node - -//composite_nodes with only output_ports -template -class composite_node , tbb::flow::tuple > : public graph_node { -public: - typedef tbb::flow::tuple< sender&... > output_ports_type; - -private: - std::unique_ptr my_output_ports; - static const size_t NUM_OUTPUTS = sizeof...(OutputTypes); - -protected: - void reset_node(reset_flags) __TBB_override {} - -public: -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - __TBB_NOINLINE_SYM composite_node( graph &g, const char *type_name = "composite_node") : graph_node(g) { - tbb::internal::fgt_composite( CODEPTR(), this, &g ); - tbb::internal::fgt_multiinput_multioutput_node_desc( this, type_name ); - } -#else - __TBB_NOINLINE_SYM composite_node( graph &g ) : graph_node(g) { - tbb::internal::fgt_composite( CODEPTR(), this, &g ); - } -#endif - - template - void set_external_ports(T&& output_ports_tuple) { - __TBB_STATIC_ASSERT(NUM_OUTPUTS == tbb::flow::tuple_size::value, "number of arguments does not match number of output ports"); - - my_output_ports = tbb::internal::make_unique(std::forward(output_ports_tuple)); - - tbb::internal::fgt_internal_output_alias_helper::alias_port( this, std::forward(output_ports_tuple)); - } - - template - void add_visible_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, true, n...); } - - template - void add_nodes(const NodeTypes&... n) { internal::add_nodes_impl(this, false, n...); } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multiinput_multioutput_node_desc( this, name ); - } -#endif - - output_ports_type& output_ports() { - __TBB_ASSERT(my_output_ports, "output ports not set, call set_external_ports to set output ports"); - return *my_output_ports; - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - void extract() __TBB_override { - __TBB_ASSERT(false, "Current composite_node implementation does not support extract"); - } -#endif - -}; // class composite_node - -#endif // __TBB_FLOW_GRAPH_CPP11_FEATURES - -namespace internal { - -template -class async_body_base: tbb::internal::no_assign { -public: - typedef Gateway gateway_type; - - async_body_base(gateway_type *gateway): my_gateway(gateway) { } - void set_gateway(gateway_type *gateway) { - my_gateway = gateway; - } - -protected: - gateway_type *my_gateway; -}; - -template -class async_body: public async_body_base { -public: - typedef async_body_base base_type; - typedef Gateway gateway_type; - - async_body(const Body &body, gateway_type *gateway) - : base_type(gateway), my_body(body) { } - - void operator()( const Input &v, Ports & ) { - my_body(v, *this->my_gateway); - } - - Body get_body() { return my_body; } - -private: - Body my_body; -}; - -} // namespace internal - -} // namespace interfaceX -namespace interface11 { - -//! Implements async node -template < typename Input, typename Output, - typename Policy = queueing_lightweight, - typename Allocator=cache_aligned_allocator > -class async_node : public multifunction_node< Input, tuple< Output >, Policy, Allocator >, public sender< Output > { - typedef multifunction_node< Input, tuple< Output >, Policy, Allocator > base_type; - typedef typename internal::multifunction_input mfn_input_type; - -public: - typedef Input input_type; - typedef Output output_type; - typedef receiver receiver_type; - typedef typename receiver_type::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - typedef receiver_gateway gateway_type; - typedef internal::async_body_base async_body_base_type; - typedef typename base_type::output_ports_type output_ports_type; - -private: - struct try_put_functor { - typedef internal::multifunction_output output_port_type; - output_port_type *port; - // TODO: pass value by copy since we do not want to block asynchronous thread. - const Output *value; - bool result; - try_put_functor(output_port_type &p, const Output &v) : port(&p), value(&v), result(false) { } - void operator()() { - result = port->try_put(*value); - } - }; - - class receiver_gateway_impl: public receiver_gateway { - public: - receiver_gateway_impl(async_node* node): my_node(node) {} - void reserve_wait() __TBB_override { - tbb::internal::fgt_async_reserve(static_cast(my_node), &my_node->my_graph); - my_node->my_graph.reserve_wait(); - } - - void release_wait() __TBB_override { - my_node->my_graph.release_wait(); - tbb::internal::fgt_async_commit(static_cast(my_node), &my_node->my_graph); - } - - //! Implements gateway_type::try_put for an external activity to submit a message to FG - bool try_put(const Output &i) __TBB_override { - return my_node->try_put_impl(i); - } - - private: - async_node* my_node; - } my_gateway; - - //The substitute of 'this' for member construction, to prevent compiler warnings - async_node* self() { return this; } - - //! Implements gateway_type::try_put for an external activity to submit a message to FG - bool try_put_impl(const Output &i) { - internal::multifunction_output &port_0 = internal::output_port<0>(*this); - internal::broadcast_cache& port_successors = port_0.successors(); - tbb::internal::fgt_async_try_put_begin(this, &port_0); - task_list tasks; - bool is_at_least_one_put_successful = port_successors.gather_successful_try_puts(i, tasks); - __TBB_ASSERT( is_at_least_one_put_successful || tasks.empty(), - "Return status is inconsistent with the method operation." ); - - while( !tasks.empty() ) { - internal::enqueue_in_graph_arena(this->my_graph, tasks.pop_front()); - } - tbb::internal::fgt_async_try_put_end(this, &port_0); - return is_at_least_one_put_successful; - } - -public: - template - __TBB_NOINLINE_SYM async_node( - graph &g, size_t concurrency, -#if __TBB_CPP11_PRESENT - Body body, __TBB_FLOW_GRAPH_PRIORITY_ARG1(Policy = Policy(), node_priority_t priority = tbb::flow::internal::no_priority) -#else - __TBB_FLOW_GRAPH_PRIORITY_ARG1(Body body, node_priority_t priority = tbb::flow::internal::no_priority) -#endif - ) : base_type( - g, concurrency, - internal::async_body - (body, &my_gateway) __TBB_FLOW_GRAPH_PRIORITY_ARG0(priority) ), my_gateway(self()) { - tbb::internal::fgt_multioutput_node_with_body<1>( - CODEPTR(), tbb::internal::FLOW_ASYNC_NODE, - &this->my_graph, static_cast *>(this), - this->output_ports(), this->my_body - ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES && __TBB_CPP11_PRESENT - template - __TBB_NOINLINE_SYM async_node(graph& g, size_t concurrency, Body body, node_priority_t priority) - : async_node(g, concurrency, body, Policy(), priority) {} -#endif // __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - __TBB_NOINLINE_SYM async_node( - const node_set& nodes, size_t concurrency, Body body, - __TBB_FLOW_GRAPH_PRIORITY_ARG1(Policy = Policy(), node_priority_t priority = tbb::flow::internal::no_priority) - ) : async_node(nodes.graph_reference(), concurrency, __TBB_FLOW_GRAPH_PRIORITY_ARG1(body, priority)) { - make_edges_in_order(nodes, *this); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES - template - __TBB_NOINLINE_SYM async_node(const node_set& nodes, size_t concurrency, Body body, node_priority_t priority) - : async_node(nodes, concurrency, body, Policy(), priority) {} -#endif // __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES -#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - - __TBB_NOINLINE_SYM async_node( const async_node &other ) : base_type(other), sender(), my_gateway(self()) { - static_cast(this->my_body->get_body_ptr())->set_gateway(&my_gateway); - static_cast(this->my_init_body->get_body_ptr())->set_gateway(&my_gateway); - - tbb::internal::fgt_multioutput_node_with_body<1>( CODEPTR(), tbb::internal::FLOW_ASYNC_NODE, - &this->my_graph, static_cast *>(this), - this->output_ports(), this->my_body ); - } - - gateway_type& gateway() { - return my_gateway; - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_multioutput_node_desc( this, name ); - } -#endif - - // Define sender< Output > - - //! Add a new successor to this node - bool register_successor( successor_type &r ) __TBB_override { - return internal::output_port<0>(*this).register_successor(r); - } - - //! Removes a successor from this node - bool remove_successor( successor_type &r ) __TBB_override { - return internal::output_port<0>(*this).remove_successor(r); - } - - template - Body copy_function_object() { - typedef internal::multifunction_body mfn_body_type; - typedef internal::async_body async_body_type; - mfn_body_type &body_ref = *this->my_body; - async_body_type ab = *static_cast(dynamic_cast< internal::multifunction_body_leaf & >(body_ref).get_body_ptr()); - return ab.get_body(); - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - //! interface to record edges for traversal & deletion - typedef typename internal::edge_container built_successors_type; - typedef typename built_successors_type::edge_list_type successor_list_type; - built_successors_type &built_successors() __TBB_override { - return internal::output_port<0>(*this).built_successors(); - } - - void internal_add_built_successor( successor_type &r ) __TBB_override { - internal::output_port<0>(*this).internal_add_built_successor(r); - } - - void internal_delete_built_successor( successor_type &r ) __TBB_override { - internal::output_port<0>(*this).internal_delete_built_successor(r); - } - - void copy_successors( successor_list_type &l ) __TBB_override { - internal::output_port<0>(*this).copy_successors(l); - } - - size_t successor_count() __TBB_override { - return internal::output_port<0>(*this).successor_count(); - } -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - -protected: - - void reset_node( reset_flags f) __TBB_override { - base_type::reset_node(f); - } -}; - -#if __TBB_PREVIEW_STREAMING_NODE -#include "internal/_flow_graph_streaming_node.h" -#endif // __TBB_PREVIEW_STREAMING_NODE - -#include "internal/_flow_graph_node_set_impl.h" - -template< typename T > -class overwrite_node : public graph_node, public receiver, public sender { -public: - typedef T input_type; - typedef T output_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename receiver::built_predecessors_type built_predecessors_type; - typedef typename sender::built_successors_type built_successors_type; - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - - __TBB_NOINLINE_SYM explicit overwrite_node(graph &g) : graph_node(g), my_buffer_is_valid(false) { - my_successors.set_owner( this ); - tbb::internal::fgt_node( CODEPTR(), tbb::internal::FLOW_OVERWRITE_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - overwrite_node(const node_set& nodes) : overwrite_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - //! Copy constructor; doesn't take anything from src; default won't work - __TBB_NOINLINE_SYM overwrite_node( const overwrite_node& src ) : - graph_node(src.my_graph), receiver(), sender(), my_buffer_is_valid(false) - { - my_successors.set_owner( this ); - tbb::internal::fgt_node( CODEPTR(), tbb::internal::FLOW_OVERWRITE_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(this) ); - } - - ~overwrite_node() {} - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - - bool register_successor( successor_type &s ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - if (my_buffer_is_valid && internal::is_graph_active( my_graph )) { - // We have a valid value that must be forwarded immediately. - bool ret = s.try_put( my_buffer ); - if ( ret ) { - // We add the successor that accepted our put - my_successors.register_successor( s ); - } else { - // In case of reservation a race between the moment of reservation and register_successor can appear, - // because failed reserve does not mean that register_successor is not ready to put a message immediately. - // We have some sort of infinite loop: reserving node tries to set pull state for the edge, - // but overwrite_node tries to return push state back. That is why we have to break this loop with task creation. - task *rtask = new ( task::allocate_additional_child_of( *( my_graph.root_task() ) ) ) - register_predecessor_task( *this, s ); - internal::spawn_in_graph_arena( my_graph, *rtask ); - } - } else { - // No valid value yet, just add as successor - my_successors.register_successor( s ); - } - return true; - } - - bool remove_successor( successor_type &s ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.remove_successor(s); - return true; - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - void internal_add_built_successor( successor_type &s) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.internal_add_built_successor(s); - } - - void internal_delete_built_successor( successor_type &s) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.internal_delete_built_successor(s); - } - - size_t successor_count() __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - return my_successors.successor_count(); - } - - void copy_successors(successor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_successors.copy_successors(v); - } - - void internal_add_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_built_predecessors.add_edge(p); - } - - void internal_delete_built_predecessor( predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_built_predecessors.delete_edge(p); - } - - size_t predecessor_count() __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - return my_built_predecessors.edge_count(); - } - - void copy_predecessors( predecessor_list_type &v ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - my_built_predecessors.copy_edges(v); - } - - void extract() __TBB_override { - my_buffer_is_valid = false; - built_successors().sender_extract(*this); - built_predecessors().receiver_extract(*this); - } - -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - - bool try_get( input_type &v ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - if ( my_buffer_is_valid ) { - v = my_buffer; - return true; - } - return false; - } - - //! Reserves an item - bool try_reserve( T &v ) __TBB_override { - return try_get(v); - } - - //! Releases the reserved item - bool try_release() __TBB_override { return true; } - - //! Consumes the reserved item - bool try_consume() __TBB_override { return true; } - - bool is_valid() { - spin_mutex::scoped_lock l( my_mutex ); - return my_buffer_is_valid; - } - - void clear() { - spin_mutex::scoped_lock l( my_mutex ); - my_buffer_is_valid = false; - } - -protected: - - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task * try_put_task( const input_type &v ) __TBB_override { - spin_mutex::scoped_lock l( my_mutex ); - return try_put_task_impl(v); - } - - task * try_put_task_impl(const input_type &v) { - my_buffer = v; - my_buffer_is_valid = true; - task * rtask = my_successors.try_put_task(v); - if (!rtask) rtask = SUCCESSFULLY_ENQUEUED; - return rtask; - } - - graph& graph_reference() const __TBB_override { - return my_graph; - } - - //! Breaks an infinite loop between the node reservation and register_successor call - struct register_predecessor_task : public graph_task { - - register_predecessor_task(predecessor_type& owner, successor_type& succ) : - o(owner), s(succ) {}; - - tbb::task* execute() __TBB_override { - if (!s.register_predecessor(o)) { - o.register_successor(s); - } - return NULL; - } - - predecessor_type& o; - successor_type& s; - }; - - spin_mutex my_mutex; - internal::broadcast_cache< input_type, null_rw_mutex > my_successors; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - internal::edge_container my_built_predecessors; -#endif - input_type my_buffer; - bool my_buffer_is_valid; - void reset_receiver(reset_flags /*f*/) __TBB_override {} - - void reset_node( reset_flags f) __TBB_override { - my_buffer_is_valid = false; - if (f&rf_clear_edges) { - my_successors.clear(); - } - } -}; // overwrite_node - -template< typename T > -class write_once_node : public overwrite_node { -public: - typedef T input_type; - typedef T output_type; - typedef overwrite_node base_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename sender::successor_type successor_type; - - //! Constructor - __TBB_NOINLINE_SYM explicit write_once_node(graph& g) : base_type(g) { - tbb::internal::fgt_node( CODEPTR(), tbb::internal::FLOW_WRITE_ONCE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - template - write_once_node(const node_set& nodes) : write_once_node(nodes.graph_reference()) { - make_edges_in_order(nodes, *this); - } -#endif - - //! Copy constructor: call base class copy constructor - __TBB_NOINLINE_SYM write_once_node( const write_once_node& src ) : base_type(src) { - tbb::internal::fgt_node( CODEPTR(), tbb::internal::FLOW_WRITE_ONCE_NODE, &(this->my_graph), - static_cast *>(this), - static_cast *>(this) ); - } - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name( const char *name ) __TBB_override { - tbb::internal::fgt_node_desc( this, name ); - } -#endif - -protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task *try_put_task( const T &v ) __TBB_override { - spin_mutex::scoped_lock l( this->my_mutex ); - return this->my_buffer_is_valid ? NULL : this->try_put_task_impl(v); - } -}; - -} // interfaceX - - using interface11::reset_flags; - using interface11::rf_reset_protocol; - using interface11::rf_reset_bodies; - using interface11::rf_clear_edges; - - using interface11::graph; - using interface11::graph_node; - using interface11::continue_msg; - - using interface11::source_node; - using interface11::function_node; - using interface11::multifunction_node; - using interface11::split_node; - using interface11::internal::output_port; - using interface11::indexer_node; - using interface11::internal::tagged_msg; - using interface11::internal::cast_to; - using interface11::internal::is_a; - using interface11::continue_node; - using interface11::overwrite_node; - using interface11::write_once_node; - using interface11::broadcast_node; - using interface11::buffer_node; - using interface11::queue_node; - using interface11::sequencer_node; - using interface11::priority_queue_node; - using interface11::limiter_node; - using namespace interface11::internal::graph_policy_namespace; - using interface11::join_node; - using interface11::input_port; - using interface11::copy_body; - using interface11::make_edge; - using interface11::remove_edge; - using interface11::internal::tag_value; -#if __TBB_FLOW_GRAPH_CPP11_FEATURES - using interface11::composite_node; -#endif - using interface11::async_node; -#if __TBB_PREVIEW_ASYNC_MSG - using interface11::async_msg; -#endif -#if __TBB_PREVIEW_STREAMING_NODE - using interface11::port_ref; - using interface11::streaming_node; -#endif // __TBB_PREVIEW_STREAMING_NODE -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES - using internal::node_priority_t; - using internal::no_priority; -#endif - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - using interface11::internal::follows; - using interface11::internal::precedes; - using interface11::internal::make_node_set; - using interface11::internal::make_edges; -#endif - -} // flow -} // tbb - -// Include deduction guides for node classes -#include "internal/_flow_graph_nodes_deduction.h" - -#undef __TBB_PFG_RESET_ARG -#undef __TBB_COMMA - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_flow_graph_H_include_area - -#if TBB_USE_THREADING_TOOLS && TBB_PREVIEW_FLOW_GRAPH_TRACE && ( __linux__ || __APPLE__ ) - #undef __TBB_NOINLINE_SYM -#endif - -#endif // __TBB_flow_graph_H +#include "../oneapi/tbb/flow_graph.h" diff --git a/src/3rd party/tbb/flow_graph_abstractions.h b/src/3rd party/tbb/flow_graph_abstractions.h index 0306763d22..cd9dc2967e 100644 --- a/src/3rd party/tbb/flow_graph_abstractions.h +++ b/src/3rd party/tbb/flow_graph_abstractions.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,40 +14,4 @@ limitations under the License. */ -#ifndef __TBB_flow_graph_abstractions_H -#define __TBB_flow_graph_abstractions_H - -namespace tbb { -namespace flow { -namespace interface11 { - -//! Pure virtual template classes that define interfaces for async communication -class graph_proxy { -public: - //! Inform a graph that messages may come from outside, to prevent premature graph completion - virtual void reserve_wait() = 0; - - //! Inform a graph that a previous call to reserve_wait is no longer in effect - virtual void release_wait() = 0; - - virtual ~graph_proxy() {} -}; - -template -class receiver_gateway : public graph_proxy { -public: - //! Type of inputing data into FG. - typedef Input input_type; - - //! Submit signal from an asynchronous activity to FG. - virtual bool try_put(const input_type&) = 0; -}; - -} //interfaceX - -using interface11::graph_proxy; -using interface11::receiver_gateway; - -} //flow -} //tbb -#endif +#include "../oneapi/tbb/flow_graph_abstractions.h" diff --git a/src/3rd party/tbb/flow_graph_opencl_node.h b/src/3rd party/tbb/flow_graph_opencl_node.h deleted file mode 100644 index c9baf7bb1a..0000000000 --- a/src/3rd party/tbb/flow_graph_opencl_node.h +++ /dev/null @@ -1,1488 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_flow_graph_opencl_node_H -#define __TBB_flow_graph_opencl_node_H - -#define __TBB_flow_graph_opencl_node_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "tbb/tbb_config.h" -#if __TBB_PREVIEW_OPENCL_NODE - -#include "flow_graph.h" - -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#else -#include -#endif - -namespace tbb { -namespace flow { - -namespace interface11 { - -template -class opencl_factory; - -namespace opencl_info { -class default_opencl_factory; -} - -template -class opencl_program; - -inline void enforce_cl_retcode(cl_int err, std::string msg) { - if (err != CL_SUCCESS) { - std::cerr << msg << "; error code: " << err << std::endl; - throw msg; - } -} - -template -T event_info(cl_event e, cl_event_info i) { - T res; - enforce_cl_retcode(clGetEventInfo(e, i, sizeof(res), &res, NULL), "Failed to get OpenCL event information"); - return res; -} - -template -T device_info(cl_device_id d, cl_device_info i) { - T res; - enforce_cl_retcode(clGetDeviceInfo(d, i, sizeof(res), &res, NULL), "Failed to get OpenCL device information"); - return res; -} - -template <> -inline std::string device_info(cl_device_id d, cl_device_info i) { - size_t required; - enforce_cl_retcode(clGetDeviceInfo(d, i, 0, NULL, &required), "Failed to get OpenCL device information"); - - char *buff = (char*)alloca(required); - enforce_cl_retcode(clGetDeviceInfo(d, i, required, buff, NULL), "Failed to get OpenCL device information"); - - return buff; -} - -template -T platform_info(cl_platform_id p, cl_platform_info i) { - T res; - enforce_cl_retcode(clGetPlatformInfo(p, i, sizeof(res), &res, NULL), "Failed to get OpenCL platform information"); - return res; -} - -template <> -inline std::string platform_info(cl_platform_id p, cl_platform_info i) { - size_t required; - enforce_cl_retcode(clGetPlatformInfo(p, i, 0, NULL, &required), "Failed to get OpenCL platform information"); - - char *buff = (char*)alloca(required); - enforce_cl_retcode(clGetPlatformInfo(p, i, required, buff, NULL), "Failed to get OpenCL platform information"); - - return buff; -} - - -class opencl_device { -public: - typedef size_t device_id_type; - enum : device_id_type { - unknown = device_id_type( -2 ), - host = device_id_type( -1 ) - }; - - opencl_device() : my_device_id( unknown ), my_cl_device_id( NULL ), my_cl_command_queue( NULL ) {} - - opencl_device( cl_device_id d_id ) : my_device_id( unknown ), my_cl_device_id( d_id ), my_cl_command_queue( NULL ) {} - - opencl_device( cl_device_id cl_d_id, device_id_type device_id ) : my_device_id( device_id ), my_cl_device_id( cl_d_id ), my_cl_command_queue( NULL ) {} - - std::string platform_profile() const { - return platform_info( platform_id(), CL_PLATFORM_PROFILE ); - } - std::string platform_version() const { - return platform_info( platform_id(), CL_PLATFORM_VERSION ); - } - std::string platform_name() const { - return platform_info( platform_id(), CL_PLATFORM_NAME ); - } - std::string platform_vendor() const { - return platform_info( platform_id(), CL_PLATFORM_VENDOR ); - } - std::string platform_extensions() const { - return platform_info( platform_id(), CL_PLATFORM_EXTENSIONS ); - } - - template - void info( cl_device_info i, T &t ) const { - t = device_info( my_cl_device_id, i ); - } - std::string version() const { - // The version string format: OpenCL - return device_info( my_cl_device_id, CL_DEVICE_VERSION ); - } - int major_version() const { - int major; - std::sscanf( version().c_str(), "OpenCL %d", &major ); - return major; - } - int minor_version() const { - int major, minor; - std::sscanf( version().c_str(), "OpenCL %d.%d", &major, &minor ); - return minor; - } - bool out_of_order_exec_mode_on_host_present() const { -#if CL_VERSION_2_0 - if ( major_version() >= 2 ) - return (device_info( my_cl_device_id, CL_DEVICE_QUEUE_ON_HOST_PROPERTIES ) & CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE) != 0; - else -#endif /* CL_VERSION_2_0 */ - return (device_info( my_cl_device_id, CL_DEVICE_QUEUE_PROPERTIES ) & CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE) != 0; - } - bool out_of_order_exec_mode_on_device_present() const { -#if CL_VERSION_2_0 - if ( major_version() >= 2 ) - return (device_info( my_cl_device_id, CL_DEVICE_QUEUE_ON_DEVICE_PROPERTIES ) & CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE) != 0; - else -#endif /* CL_VERSION_2_0 */ - return false; - } - std::array max_work_item_sizes() const { - return device_info>( my_cl_device_id, CL_DEVICE_MAX_WORK_ITEM_SIZES ); - } - size_t max_work_group_size() const { - return device_info( my_cl_device_id, CL_DEVICE_MAX_WORK_GROUP_SIZE ); - } - bool built_in_kernel_available( const std::string& k ) const { - const std::string semi = ";"; - // Added semicolumns to force an exact match (to avoid a partial match, e.g. "add" is partly matched with "madd"). - return (semi + built_in_kernels() + semi).find( semi + k + semi ) != std::string::npos; - } - std::string built_in_kernels() const { - return device_info( my_cl_device_id, CL_DEVICE_BUILT_IN_KERNELS ); - } - std::string name() const { - return device_info( my_cl_device_id, CL_DEVICE_NAME ); - } - cl_bool available() const { - return device_info( my_cl_device_id, CL_DEVICE_AVAILABLE ); - } - cl_bool compiler_available() const { - return device_info( my_cl_device_id, CL_DEVICE_COMPILER_AVAILABLE ); - } - cl_bool linker_available() const { - return device_info( my_cl_device_id, CL_DEVICE_LINKER_AVAILABLE ); - } - bool extension_available( const std::string &ext ) const { - const std::string space = " "; - // Added space to force an exact match (to avoid a partial match, e.g. "ext" is partly matched with "ext2"). - return (space + extensions() + space).find( space + ext + space ) != std::string::npos; - } - std::string extensions() const { - return device_info( my_cl_device_id, CL_DEVICE_EXTENSIONS ); - } - - cl_device_type type() const { - return device_info( my_cl_device_id, CL_DEVICE_TYPE ); - } - - std::string vendor() const { - return device_info( my_cl_device_id, CL_DEVICE_VENDOR ); - } - - cl_uint address_bits() const { - return device_info( my_cl_device_id, CL_DEVICE_ADDRESS_BITS ); - } - - cl_device_id device_id() const { - return my_cl_device_id; - } - - cl_command_queue command_queue() const { - return my_cl_command_queue; - } - - void set_command_queue( cl_command_queue cmd_queue ) { - my_cl_command_queue = cmd_queue; - } - - cl_platform_id platform_id() const { - return device_info( my_cl_device_id, CL_DEVICE_PLATFORM ); - } - -private: - - device_id_type my_device_id; - cl_device_id my_cl_device_id; - cl_command_queue my_cl_command_queue; - - friend bool operator==(opencl_device d1, opencl_device d2) { return d1.my_cl_device_id == d2.my_cl_device_id; } - - template - friend class opencl_factory; - template - friend class opencl_memory; - template - friend class opencl_program; - -#if TBB_USE_ASSERT - template - friend class opencl_buffer; -#endif -}; - -class opencl_device_list { - typedef std::vector container_type; -public: - typedef container_type::iterator iterator; - typedef container_type::const_iterator const_iterator; - typedef container_type::size_type size_type; - - opencl_device_list() {} - opencl_device_list( std::initializer_list il ) : my_container( il ) {} - - void add( opencl_device d ) { my_container.push_back( d ); } - size_type size() const { return my_container.size(); } - bool empty() const { return my_container.empty(); } - iterator begin() { return my_container.begin(); } - iterator end() { return my_container.end(); } - const_iterator begin() const { return my_container.begin(); } - const_iterator end() const { return my_container.end(); } - const_iterator cbegin() const { return my_container.cbegin(); } - const_iterator cend() const { return my_container.cend(); } - -private: - container_type my_container; -}; - -namespace internal { - -// Retrieve all OpenCL devices from machine -inline opencl_device_list find_available_devices() { - opencl_device_list opencl_devices; - - cl_uint num_platforms; - enforce_cl_retcode(clGetPlatformIDs(0, NULL, &num_platforms), "clGetPlatformIDs failed"); - - std::vector platforms(num_platforms); - enforce_cl_retcode(clGetPlatformIDs(num_platforms, platforms.data(), NULL), "clGetPlatformIDs failed"); - - cl_uint num_devices; - std::vector::iterator platforms_it = platforms.begin(); - cl_uint num_all_devices = 0; - while (platforms_it != platforms.end()) { - cl_int err = clGetDeviceIDs(*platforms_it, CL_DEVICE_TYPE_ALL, 0, NULL, &num_devices); - if (err == CL_DEVICE_NOT_FOUND) { - platforms_it = platforms.erase(platforms_it); - } - else { - enforce_cl_retcode(err, "clGetDeviceIDs failed"); - num_all_devices += num_devices; - ++platforms_it; - } - } - - std::vector devices(num_all_devices); - std::vector::iterator devices_it = devices.begin(); - for (auto p = platforms.begin(); p != platforms.end(); ++p) { - enforce_cl_retcode(clGetDeviceIDs((*p), CL_DEVICE_TYPE_ALL, (cl_uint)std::distance(devices_it, devices.end()), &*devices_it, &num_devices), "clGetDeviceIDs failed"); - devices_it += num_devices; - } - - for (auto d = devices.begin(); d != devices.end(); ++d) { - opencl_devices.add(opencl_device((*d))); - } - - return opencl_devices; -} - -} // namespace internal - -// TODO: consider this namespace as public API -namespace opencl_info { - - inline const opencl_device_list& available_devices() { - // Static storage for all available OpenCL devices on machine - static const opencl_device_list my_devices = internal::find_available_devices(); - return my_devices; - } - -} // namespace opencl_info - - -class callback_base : tbb::internal::no_copy { -public: - virtual void call() = 0; - virtual ~callback_base() {} -}; - -template -class callback : public callback_base { - Callback my_callback; - T my_data; -public: - callback( Callback c, const T& t ) : my_callback( c ), my_data( t ) {} - - void call() __TBB_override { - my_callback( my_data ); - } -}; - -template -class opencl_async_msg : public async_msg { -public: - typedef T value_type; - - opencl_async_msg() : my_callback_flag_ptr( std::make_shared< tbb::atomic>() ) { - my_callback_flag_ptr->store(false); - } - - explicit opencl_async_msg( const T& data ) : my_data(data), my_callback_flag_ptr( std::make_shared>() ) { - my_callback_flag_ptr->store(false); - } - - opencl_async_msg( const T& data, cl_event event ) : my_data(data), my_event(event), my_is_event(true), my_callback_flag_ptr( std::make_shared>() ) { - my_callback_flag_ptr->store(false); - enforce_cl_retcode( clRetainEvent( my_event ), "Failed to retain an event" ); - } - - T& data( bool wait = true ) { - if ( my_is_event && wait ) { - enforce_cl_retcode( clWaitForEvents( 1, &my_event ), "Failed to wait for an event" ); - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to release an event" ); - my_is_event = false; - } - return my_data; - } - - const T& data( bool wait = true ) const { - if ( my_is_event && wait ) { - enforce_cl_retcode( clWaitForEvents( 1, &my_event ), "Failed to wait for an event" ); - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to release an event" ); - my_is_event = false; - } - return my_data; - } - - opencl_async_msg( const opencl_async_msg &dmsg ) : async_msg(dmsg), - my_data(dmsg.my_data), my_event(dmsg.my_event), my_is_event( dmsg.my_is_event ), - my_callback_flag_ptr(dmsg.my_callback_flag_ptr) - { - if ( my_is_event ) - enforce_cl_retcode( clRetainEvent( my_event ), "Failed to retain an event" ); - } - - opencl_async_msg( opencl_async_msg &&dmsg ) : async_msg(std::move(dmsg)), - my_data(std::move(dmsg.my_data)), my_event(dmsg.my_event), my_is_event(dmsg.my_is_event), - my_callback_flag_ptr( std::move(dmsg.my_callback_flag_ptr) ) - { - dmsg.my_is_event = false; - } - - opencl_async_msg& operator=(const opencl_async_msg &dmsg) { - async_msg::operator =(dmsg); - - // Release original event - if ( my_is_event ) - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to retain an event" ); - - my_data = dmsg.my_data; - my_event = dmsg.my_event; - my_is_event = dmsg.my_is_event; - - // Retain copied event - if ( my_is_event ) - enforce_cl_retcode( clRetainEvent( my_event ), "Failed to retain an event" ); - - my_callback_flag_ptr = dmsg.my_callback_flag_ptr; - return *this; - } - - ~opencl_async_msg() { - if ( my_is_event ) - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to release an event" ); - } - - cl_event const * get_event() const { return my_is_event ? &my_event : NULL; } - void set_event( cl_event e ) const { - if ( my_is_event ) { - cl_command_queue cq = event_info( my_event, CL_EVENT_COMMAND_QUEUE ); - if ( cq != event_info( e, CL_EVENT_COMMAND_QUEUE ) ) - enforce_cl_retcode( clFlush( cq ), "Failed to flush an OpenCL command queue" ); - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to release an event" ); - } - my_is_event = true; - my_event = e; - clRetainEvent( my_event ); - } - - void clear_event() const { - if ( my_is_event ) { - enforce_cl_retcode( clFlush( event_info( my_event, CL_EVENT_COMMAND_QUEUE ) ), "Failed to flush an OpenCL command queue" ); - enforce_cl_retcode( clReleaseEvent( my_event ), "Failed to release an event" ); - } - my_is_event = false; - } - - template - void register_callback( Callback c ) const { - __TBB_ASSERT( my_is_event, "The OpenCL event is not set" ); - enforce_cl_retcode( clSetEventCallback( my_event, CL_COMPLETE, register_callback_func, new callback( c, my_data ) ), "Failed to set an OpenCL callback" ); - } - - operator T&() { return data(); } - operator const T&() const { return data(); } - -protected: - // Overridden in this derived class to inform that - // async calculation chain is over - void finalize() const __TBB_override { - receive_if_memory_object(*this); - if (! my_callback_flag_ptr->fetch_and_store(true)) { - opencl_async_msg a(*this); - if (my_is_event) { - register_callback([a](const T& t) mutable { - a.set(t); - }); - } - else { - a.set(my_data); - } - } - clear_event(); - } - -private: - static void CL_CALLBACK register_callback_func( cl_event, cl_int event_command_exec_status, void *data ) { - tbb::internal::suppress_unused_warning( event_command_exec_status ); - __TBB_ASSERT( event_command_exec_status == CL_COMPLETE, NULL ); - __TBB_ASSERT( data, NULL ); - callback_base *c = static_cast(data); - c->call(); - delete c; - } - - T my_data; - mutable cl_event my_event; - mutable bool my_is_event = false; - - std::shared_ptr< tbb::atomic > my_callback_flag_ptr; -}; - -template -K key_from_message( const opencl_async_msg &dmsg ) { - using tbb::flow::key_from_message; - const T &t = dmsg.data( false ); - __TBB_STATIC_ASSERT( true, "" ); - return key_from_message( t ); -} - -template -class opencl_memory { -public: - opencl_memory() {} - opencl_memory( Factory &f ) : my_host_ptr( NULL ), my_factory( &f ), my_sending_event_present( false ) { - my_curr_device_id = my_factory->devices().begin()->my_device_id; - } - - ~opencl_memory() { - if ( my_sending_event_present ) enforce_cl_retcode( clReleaseEvent( my_sending_event ), "Failed to release an event for the OpenCL buffer" ); - enforce_cl_retcode( clReleaseMemObject( my_cl_mem ), "Failed to release an memory object" ); - } - - cl_mem get_cl_mem() const { - return my_cl_mem; - } - - void* get_host_ptr() { - if ( !my_host_ptr ) { - opencl_async_msg d = receive( NULL ); - d.data(); - __TBB_ASSERT( d.data() == my_host_ptr, NULL ); - } - return my_host_ptr; - } - - Factory *factory() const { return my_factory; } - - opencl_async_msg receive(const cl_event *e) { - opencl_async_msg d; - if (e) { - d = opencl_async_msg(my_host_ptr, *e); - } else { - d = opencl_async_msg(my_host_ptr); - } - - // Concurrent receives are prohibited so we do not worry about synchronization. - if (my_curr_device_id.load() != opencl_device::host) { - map_memory(*my_factory->devices().begin(), d); - my_curr_device_id.store(opencl_device::host); - my_host_ptr = d.data(false); - } - // Release the sending event - if (my_sending_event_present) { - enforce_cl_retcode(clReleaseEvent(my_sending_event), "Failed to release an event"); - my_sending_event_present = false; - } - return d; - } - - opencl_async_msg send(opencl_device device, const cl_event *e) { - opencl_device::device_id_type device_id = device.my_device_id; - if (!my_factory->is_same_context(my_curr_device_id.load(), device_id)) { - { - tbb::spin_mutex::scoped_lock lock(my_sending_lock); - if (!my_factory->is_same_context(my_curr_device_id.load(), device_id)) { - __TBB_ASSERT(my_host_ptr, "The buffer has not been mapped"); - opencl_async_msg d(my_host_ptr); - my_factory->enqueue_unmap_buffer(device, *this, d); - my_sending_event = *d.get_event(); - my_sending_event_present = true; - enforce_cl_retcode(clRetainEvent(my_sending_event), "Failed to retain an event"); - my_host_ptr = NULL; - my_curr_device_id.store(device_id); - } - } - __TBB_ASSERT(my_sending_event_present, NULL); - } - - // !e means that buffer has come from the host - if (!e && my_sending_event_present) e = &my_sending_event; - - __TBB_ASSERT(!my_host_ptr, "The buffer has not been unmapped"); - return e ? opencl_async_msg(NULL, *e) : opencl_async_msg(NULL); - } - - virtual void map_memory( opencl_device, opencl_async_msg & ) = 0; -protected: - cl_mem my_cl_mem; - tbb::atomic my_curr_device_id; - void* my_host_ptr; - Factory *my_factory; - - tbb::spin_mutex my_sending_lock; - bool my_sending_event_present; - cl_event my_sending_event; -}; - -template -class opencl_buffer_impl : public opencl_memory { - size_t my_size; -public: - opencl_buffer_impl( size_t size, Factory& f ) : opencl_memory( f ), my_size( size ) { - cl_int err; - this->my_cl_mem = clCreateBuffer( this->my_factory->context(), CL_MEM_ALLOC_HOST_PTR, size, NULL, &err ); - enforce_cl_retcode( err, "Failed to create an OpenCL buffer" ); - } - - // The constructor for subbuffers. - opencl_buffer_impl( cl_mem m, size_t index, size_t size, Factory& f ) : opencl_memory( f ), my_size( size ) { - cl_int err; - cl_buffer_region region = { index, size }; - this->my_cl_mem = clCreateSubBuffer( m, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err ); - enforce_cl_retcode( err, "Failed to create an OpenCL subbuffer" ); - } - - size_t size() const { - return my_size; - } - - void map_memory( opencl_device device, opencl_async_msg &dmsg ) __TBB_override { - this->my_factory->enqueue_map_buffer( device, *this, dmsg ); - } - -#if TBB_USE_ASSERT - template - friend class opencl_buffer; -#endif -}; - -enum access_type { - read_write, - write_only, - read_only -}; - -template -class opencl_subbuffer; - -template -class opencl_buffer { -public: - typedef cl_mem native_object_type; - typedef opencl_buffer memory_object_type; - typedef Factory opencl_factory_type; - - template using iterator = T*; - - template - iterator access() const { - T* ptr = (T*)my_impl->get_host_ptr(); - __TBB_ASSERT( ptr, NULL ); - return iterator( ptr ); - } - - T* data() const { return &access()[0]; } - - template - iterator begin() const { return access(); } - - template - iterator end() const { return access()+my_impl->size()/sizeof(T); } - - size_t size() const { return my_impl->size()/sizeof(T); } - - T& operator[] ( ptrdiff_t k ) { return begin()[k]; } - - opencl_buffer() {} - opencl_buffer( size_t size ); - opencl_buffer( Factory &f, size_t size ) : my_impl( std::make_shared( size*sizeof(T), f ) ) {} - - cl_mem native_object() const { - return my_impl->get_cl_mem(); - } - - const opencl_buffer& memory_object() const { - return *this; - } - - void send( opencl_device device, opencl_async_msg &dependency ) const { - __TBB_ASSERT( dependency.data( /*wait = */false ) == *this, NULL ); - opencl_async_msg d = my_impl->send( device, dependency.get_event() ); - const cl_event *e = d.get_event(); - if ( e ) dependency.set_event( *e ); - else dependency.clear_event(); - } - void receive( const opencl_async_msg &dependency ) const { - __TBB_ASSERT( dependency.data( /*wait = */false ) == *this, NULL ); - opencl_async_msg d = my_impl->receive( dependency.get_event() ); - const cl_event *e = d.get_event(); - if ( e ) dependency.set_event( *e ); - else dependency.clear_event(); - } - - opencl_subbuffer subbuffer( size_t index, size_t size ) const; -private: - // The constructor for subbuffers. - opencl_buffer( Factory &f, cl_mem m, size_t index, size_t size ) : my_impl( std::make_shared( m, index*sizeof(T), size*sizeof(T), f ) ) {} - - typedef opencl_buffer_impl impl_type; - - std::shared_ptr my_impl; - - friend bool operator==(const opencl_buffer &lhs, const opencl_buffer &rhs) { - return lhs.my_impl == rhs.my_impl; - } - - template - friend class opencl_factory; - template - friend class opencl_subbuffer; -}; - -template -class opencl_subbuffer : public opencl_buffer { - opencl_buffer my_owner; -public: - opencl_subbuffer() {} - opencl_subbuffer( const opencl_buffer &owner, size_t index, size_t size ) : - opencl_buffer( *owner.my_impl->factory(), owner.native_object(), index, size ), my_owner( owner ) {} -}; - -template -opencl_subbuffer opencl_buffer::subbuffer( size_t index, size_t size ) const { - return opencl_subbuffer( *this, index, size ); -} - - -#define is_typedef(type) \ - template \ - struct is_##type { \ - template \ - static std::true_type check( typename C::type* ); \ - template \ - static std::false_type check( ... ); \ - \ - static const bool value = decltype(check(0))::value; \ - } - -is_typedef( native_object_type ); -is_typedef( memory_object_type ); - -template -typename std::enable_if::value, typename T::native_object_type>::type get_native_object( const T &t ) { - return t.native_object(); -} - -template -typename std::enable_if::value, T>::type get_native_object( T t ) { - return t; -} - -// send_if_memory_object checks if the T type has memory_object_type and call the send method for the object. -template -typename std::enable_if::value>::type send_if_memory_object( opencl_device device, opencl_async_msg &dmsg ) { - const T &t = dmsg.data( false ); - typedef typename T::memory_object_type mem_obj_t; - mem_obj_t mem_obj = t.memory_object(); - opencl_async_msg d( mem_obj ); - if ( dmsg.get_event() ) d.set_event( *dmsg.get_event() ); - mem_obj.send( device, d ); - if ( d.get_event() ) dmsg.set_event( *d.get_event() ); -} - -template -typename std::enable_if::value>::type send_if_memory_object( opencl_device device, T &t ) { - typedef typename T::memory_object_type mem_obj_t; - mem_obj_t mem_obj = t.memory_object(); - opencl_async_msg dmsg( mem_obj ); - mem_obj.send( device, dmsg ); -} - -template -typename std::enable_if::value>::type send_if_memory_object( opencl_device, T& ) {}; - -// receive_if_memory_object checks if the T type has memory_object_type and call the receive method for the object. -template -typename std::enable_if::value>::type receive_if_memory_object( const opencl_async_msg &dmsg ) { - const T &t = dmsg.data( false ); - typedef typename T::memory_object_type mem_obj_t; - mem_obj_t mem_obj = t.memory_object(); - opencl_async_msg d( mem_obj ); - if ( dmsg.get_event() ) d.set_event( *dmsg.get_event() ); - mem_obj.receive( d ); - if ( d.get_event() ) dmsg.set_event( *d.get_event() ); -} - -template -typename std::enable_if::value>::type receive_if_memory_object( const T& ) {} - -class opencl_range { -public: - typedef size_t range_index_type; - typedef std::array nd_range_type; - - template , typename L = std::initializer_list, - typename = typename std::enable_if::type, opencl_range>::value>::type> - opencl_range(G&& global_work = std::initializer_list({ 0 }), L&& local_work = std::initializer_list({ 0, 0, 0 })) { - auto g_it = global_work.begin(); - auto l_it = local_work.begin(); - my_global_work_size = { size_t(-1), size_t(-1), size_t(-1) }; - // my_local_work_size is still uninitialized - for (int s = 0; s < 3 && g_it != global_work.end(); ++g_it, ++l_it, ++s) { - __TBB_ASSERT(l_it != local_work.end(), "global_work & local_work must have same size"); - my_global_work_size[s] = *g_it; - my_local_work_size[s] = *l_it; - } - } - - const nd_range_type& global_range() const { return my_global_work_size; } - const nd_range_type& local_range() const { return my_local_work_size; } - -private: - nd_range_type my_global_work_size; - nd_range_type my_local_work_size; -}; - -template -class opencl_factory { -public: - template using async_msg_type = opencl_async_msg>; - typedef opencl_device device_type; - - class kernel : tbb::internal::no_assign { - public: - kernel( const kernel& k ) : my_factory( k.my_factory ) { - // Clone my_cl_kernel via opencl_program - size_t ret_size = 0; - - std::vector kernel_name; - for ( size_t curr_size = 32;; curr_size <<= 1 ) { - kernel_name.resize( curr_size <<= 1 ); - enforce_cl_retcode( clGetKernelInfo( k.my_cl_kernel, CL_KERNEL_FUNCTION_NAME, curr_size, kernel_name.data(), &ret_size ), "Failed to get kernel info" ); - if ( ret_size < curr_size ) break; - } - - cl_program program; - enforce_cl_retcode( clGetKernelInfo( k.my_cl_kernel, CL_KERNEL_PROGRAM, sizeof(program), &program, &ret_size ), "Failed to get kernel info" ); - __TBB_ASSERT( ret_size == sizeof(program), NULL ); - - my_cl_kernel = opencl_program< factory_type >( my_factory, program ).get_cl_kernel( kernel_name.data() ); - } - - ~kernel() { - enforce_cl_retcode( clReleaseKernel( my_cl_kernel ), "Failed to release a kernel" ); - } - - private: - typedef opencl_factory factory_type; - - kernel( const cl_kernel& k, factory_type& f ) : my_cl_kernel( k ), my_factory( f ) {} - - // Data - cl_kernel my_cl_kernel; - factory_type& my_factory; - - template - friend class opencl_factory; - - template - friend class opencl_program; - }; - - typedef kernel kernel_type; - - // 'range_type' enables kernel_executor with range support - // it affects expectations for enqueue_kernel(.....) interface method - typedef opencl_range range_type; - - opencl_factory() {} - ~opencl_factory() { - if ( my_devices.size() ) { - for ( auto d = my_devices.begin(); d != my_devices.end(); ++d ) { - enforce_cl_retcode( clReleaseCommandQueue( (*d).my_cl_command_queue ), "Failed to release a command queue" ); - } - enforce_cl_retcode( clReleaseContext( my_cl_context ), "Failed to release a context" ); - } - } - - bool init( const opencl_device_list &device_list ) { - tbb::spin_mutex::scoped_lock lock( my_devices_mutex ); - if ( !my_devices.size() ) { - my_devices = device_list; - return true; - } - return false; - } - - -private: - template - void enqueue_map_buffer( opencl_device device, opencl_buffer_impl &buffer, opencl_async_msg& dmsg ) { - cl_event const* e1 = dmsg.get_event(); - cl_event e2; - cl_int err; - void *ptr = clEnqueueMapBuffer( device.my_cl_command_queue, buffer.get_cl_mem(), false, CL_MAP_READ | CL_MAP_WRITE, 0, buffer.size(), - e1 == NULL ? 0 : 1, e1, &e2, &err ); - enforce_cl_retcode( err, "Failed to map a buffer" ); - dmsg.data( false ) = ptr; - dmsg.set_event( e2 ); - enforce_cl_retcode( clReleaseEvent( e2 ), "Failed to release an event" ); - } - - - template - void enqueue_unmap_buffer( opencl_device device, opencl_memory &memory, opencl_async_msg& dmsg ) { - cl_event const* e1 = dmsg.get_event(); - cl_event e2; - enforce_cl_retcode( - clEnqueueUnmapMemObject( device.my_cl_command_queue, memory.get_cl_mem(), memory.get_host_ptr(), e1 == NULL ? 0 : 1, e1, &e2 ), - "Failed to unmap a buffer" ); - dmsg.set_event( e2 ); - enforce_cl_retcode( clReleaseEvent( e2 ), "Failed to release an event" ); - } - - // --------- Kernel argument & event list helpers --------- // - template - void process_one_arg( const kernel_type& kernel, std::array&, int&, int& place, const T& t ) { - auto p = get_native_object(t); - enforce_cl_retcode( clSetKernelArg(kernel.my_cl_kernel, place++, sizeof(p), &p), "Failed to set a kernel argument" ); - } - - template - void process_one_arg( const kernel_type& kernel, std::array& events, int& num_events, int& place, const opencl_async_msg& msg ) { - __TBB_ASSERT((static_cast::size_type>(num_events) < events.size()), NULL); - - const cl_event * const e = msg.get_event(); - if (e != NULL) { - events[num_events++] = *e; - } - - process_one_arg( kernel, events, num_events, place, msg.data(false) ); - } - - template - void process_arg_list( const kernel_type& kernel, std::array& events, int& num_events, int& place, const T& t, const Rest&... args ) { - process_one_arg( kernel, events, num_events, place, t ); - process_arg_list( kernel, events, num_events, place, args... ); - } - - template - void process_arg_list( const kernel_type&, std::array&, int&, int& ) {} - // ------------------------------------------- // - template - void update_one_arg( cl_event, T& ) {} - - template - void update_one_arg( cl_event e, opencl_async_msg& msg ) { - msg.set_event( e ); - } - - template - void update_arg_list( cl_event e, T& t, Rest&... args ) { - update_one_arg( e, t ); - update_arg_list( e, args... ); - } - - void update_arg_list( cl_event ) {} - // ------------------------------------------- // -public: - template - void send_kernel( opencl_device device, const kernel_type& kernel, const range_type& work_size, Args&... args ) { - std::array events; - int num_events = 0; - int place = 0; - process_arg_list( kernel, events, num_events, place, args... ); - - const cl_event e = send_kernel_impl( device, kernel.my_cl_kernel, work_size, num_events, events.data() ); - - update_arg_list(e, args...); - - // Release our own reference to cl_event - enforce_cl_retcode( clReleaseEvent(e), "Failed to release an event" ); - } - - // ------------------------------------------- // - template - void send_data(opencl_device device, T& t, Rest&... args) { - send_if_memory_object( device, t ); - send_data( device, args... ); - } - - void send_data(opencl_device) {} - // ------------------------------------------- // - -private: - cl_event send_kernel_impl( opencl_device device, const cl_kernel& kernel, - const range_type& work_size, cl_uint num_events, cl_event* event_list ) { - const typename range_type::nd_range_type g_offset = { { 0, 0, 0 } }; - const typename range_type::nd_range_type& g_size = work_size.global_range(); - const typename range_type::nd_range_type& l_size = work_size.local_range(); - cl_uint s; - for ( s = 1; s < 3 && g_size[s] != size_t(-1); ++s) {} - cl_event event; - enforce_cl_retcode( - clEnqueueNDRangeKernel( device.my_cl_command_queue, kernel, s, - g_offset.data(), g_size.data(), l_size[0] ? l_size.data() : NULL, num_events, num_events ? event_list : NULL, &event ), - "Failed to enqueue a kernel" ); - return event; - } - - // ------------------------------------------- // - template - bool get_event_from_one_arg( cl_event&, const T& ) { - return false; - } - - template - bool get_event_from_one_arg( cl_event& e, const opencl_async_msg& msg) { - cl_event const *e_ptr = msg.get_event(); - - if ( e_ptr != NULL ) { - e = *e_ptr; - return true; - } - - return false; - } - - template - bool get_event_from_args( cl_event& e, const T& t, const Rest&... args ) { - if ( get_event_from_one_arg( e, t ) ) { - return true; - } - - return get_event_from_args( e, args... ); - } - - bool get_event_from_args( cl_event& ) { - return false; - } - // ------------------------------------------- // - - struct finalize_fn : tbb::internal::no_assign { - virtual ~finalize_fn() {} - virtual void operator() () {} - }; - - template - struct finalize_fn_leaf : public finalize_fn { - Fn my_fn; - finalize_fn_leaf(Fn fn) : my_fn(fn) {} - void operator() () __TBB_override { my_fn(); } - }; - - static void CL_CALLBACK finalize_callback(cl_event, cl_int event_command_exec_status, void *data) { - tbb::internal::suppress_unused_warning(event_command_exec_status); - __TBB_ASSERT(event_command_exec_status == CL_COMPLETE, NULL); - - finalize_fn * const fn_ptr = static_cast(data); - __TBB_ASSERT(fn_ptr != NULL, "Invalid finalize function pointer"); - (*fn_ptr)(); - - // Function pointer was created by 'new' & this callback must be called once only - delete fn_ptr; - } -public: - template - void finalize( opencl_device device, FinalizeFn fn, Args&... args ) { - cl_event e; - - if ( get_event_from_args( e, args... ) ) { - enforce_cl_retcode( clSetEventCallback( e, CL_COMPLETE, finalize_callback, - new finalize_fn_leaf(fn) ), "Failed to set a callback" ); - } - - enforce_cl_retcode( clFlush( device.my_cl_command_queue ), "Failed to flush an OpenCL command queue" ); - } - - const opencl_device_list& devices() { - std::call_once( my_once_flag, &opencl_factory::init_once, this ); - return my_devices; - } - -private: - bool is_same_context( opencl_device::device_id_type d1, opencl_device::device_id_type d2 ) { - __TBB_ASSERT( d1 != opencl_device::unknown && d2 != opencl_device::unknown, NULL ); - // Currently, factory supports only one context so if the both devices are not host it means the are in the same context. - if ( d1 != opencl_device::host && d2 != opencl_device::host ) - return true; - return d1 == d2; - } -private: - opencl_factory( const opencl_factory& ); - opencl_factory& operator=(const opencl_factory&); - - cl_context context() { - std::call_once( my_once_flag, &opencl_factory::init_once, this ); - return my_cl_context; - } - - void init_once() { - { - tbb::spin_mutex::scoped_lock lock(my_devices_mutex); - if (!my_devices.size()) - my_devices = DeviceFilter()( opencl_info::available_devices() ); - } - - enforce_cl_retcode(my_devices.size() ? CL_SUCCESS : CL_INVALID_DEVICE, "No devices in the device list"); - cl_platform_id platform_id = my_devices.begin()->platform_id(); - for (opencl_device_list::iterator it = ++my_devices.begin(); it != my_devices.end(); ++it) - enforce_cl_retcode(it->platform_id() == platform_id ? CL_SUCCESS : CL_INVALID_PLATFORM, "All devices should be in the same platform"); - - std::vector cl_device_ids; - for (auto d = my_devices.begin(); d != my_devices.end(); ++d) { - cl_device_ids.push_back((*d).my_cl_device_id); - } - - cl_context_properties context_properties[3] = { CL_CONTEXT_PLATFORM, (cl_context_properties)platform_id, (cl_context_properties)NULL }; - cl_int err; - cl_context ctx = clCreateContext(context_properties, - (cl_uint)cl_device_ids.size(), - cl_device_ids.data(), - NULL, NULL, &err); - enforce_cl_retcode(err, "Failed to create context"); - my_cl_context = ctx; - - size_t device_counter = 0; - for (auto d = my_devices.begin(); d != my_devices.end(); d++) { - (*d).my_device_id = device_counter++; - cl_int err2; - cl_command_queue cq; -#if CL_VERSION_2_0 - if ((*d).major_version() >= 2) { - if ((*d).out_of_order_exec_mode_on_host_present()) { - cl_queue_properties props[] = { CL_QUEUE_PROPERTIES, CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE, 0 }; - cq = clCreateCommandQueueWithProperties(ctx, (*d).my_cl_device_id, props, &err2); - } else { - cl_queue_properties props[] = { 0 }; - cq = clCreateCommandQueueWithProperties(ctx, (*d).my_cl_device_id, props, &err2); - } - } else -#endif - { - cl_command_queue_properties props = (*d).out_of_order_exec_mode_on_host_present() ? CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE : 0; - // Suppress "declared deprecated" warning for the next line. -#if __TBB_GCC_WARNING_SUPPRESSION_PRESENT -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#endif -#if _MSC_VER || __INTEL_COMPILER -#pragma warning( push ) -#if __INTEL_COMPILER -#pragma warning (disable: 1478) -#else -#pragma warning (disable: 4996) -#endif -#endif - cq = clCreateCommandQueue(ctx, (*d).my_cl_device_id, props, &err2); -#if _MSC_VER || __INTEL_COMPILER -#pragma warning( pop ) -#endif -#if __TBB_GCC_WARNING_SUPPRESSION_PRESENT -#pragma GCC diagnostic pop -#endif - } - enforce_cl_retcode(err2, "Failed to create command queue"); - (*d).my_cl_command_queue = cq; - } - } - - std::once_flag my_once_flag; - opencl_device_list my_devices; - cl_context my_cl_context; - - tbb::spin_mutex my_devices_mutex; - - template - friend class opencl_program; - template - friend class opencl_buffer_impl; - template - friend class opencl_memory; -}; // class opencl_factory - -// TODO: consider this namespace as public API -namespace opencl_info { - -// Default types - -template -struct default_device_selector { - opencl_device operator()(Factory& f) { - __TBB_ASSERT(!f.devices().empty(), "No available devices"); - return *(f.devices().begin()); - } -}; - -struct default_device_filter { - opencl_device_list operator()(const opencl_device_list &devices) { - opencl_device_list dl; - cl_platform_id platform_id = devices.begin()->platform_id(); - for (opencl_device_list::const_iterator it = devices.cbegin(); it != devices.cend(); ++it) { - if (it->platform_id() == platform_id) { - dl.add(*it); - } - } - return dl; - } -}; - -class default_opencl_factory : public opencl_factory < default_device_filter >, tbb::internal::no_copy { -public: - template using async_msg_type = opencl_async_msg; - - friend default_opencl_factory& default_factory(); - -private: - default_opencl_factory() = default; -}; - -inline default_opencl_factory& default_factory() { - static default_opencl_factory default_factory; - return default_factory; -} - -} // namespace opencl_info - -template -opencl_buffer::opencl_buffer( size_t size ) : my_impl( std::make_shared( size*sizeof(T), opencl_info::default_factory() ) ) {} - - -enum class opencl_program_type { - SOURCE, - PRECOMPILED, - SPIR -}; - -template -class opencl_program : tbb::internal::no_assign { -public: - typedef typename Factory::kernel_type kernel_type; - - opencl_program( Factory& factory, opencl_program_type type, const std::string& program_name ) : my_factory( factory ), my_type(type) , my_arg_str( program_name) {} - opencl_program( Factory& factory, const char* program_name ) : opencl_program( factory, std::string( program_name ) ) {} - opencl_program( Factory& factory, const std::string& program_name ) : opencl_program( factory, opencl_program_type::SOURCE, program_name ) {} - - opencl_program( opencl_program_type type, const std::string& program_name ) : opencl_program( opencl_info::default_factory(), type, program_name ) {} - opencl_program( const char* program_name ) : opencl_program( opencl_info::default_factory(), program_name ) {} - opencl_program( const std::string& program_name ) : opencl_program( opencl_info::default_factory(), program_name ) {} - opencl_program( opencl_program_type type ) : opencl_program( opencl_info::default_factory(), type ) {} - - opencl_program( const opencl_program &src ) : my_factory( src.my_factory ), my_type( src.type ), my_arg_str( src.my_arg_str ), my_cl_program( src.my_cl_program ) { - // Set my_do_once_flag to the called state. - std::call_once( my_do_once_flag, [](){} ); - } - - kernel_type get_kernel( const std::string& k ) const { - return kernel_type( get_cl_kernel(k), my_factory ); - } - -private: - opencl_program( Factory& factory, cl_program program ) : my_factory( factory ), my_cl_program( program ) { - // Set my_do_once_flag to the called state. - std::call_once( my_do_once_flag, [](){} ); - } - - cl_kernel get_cl_kernel( const std::string& k ) const { - std::call_once( my_do_once_flag, [this, &k](){ this->init( k ); } ); - cl_int err; - cl_kernel kernel = clCreateKernel( my_cl_program, k.c_str(), &err ); - enforce_cl_retcode( err, std::string( "Failed to create kernel: " ) + k ); - return kernel; - } - - class file_reader { - public: - file_reader( const std::string& filepath ) { - std::ifstream file_descriptor( filepath, std::ifstream::binary ); - if ( !file_descriptor.is_open() ) { - std::string str = std::string( "Could not open file: " ) + filepath; - std::cerr << str << std::endl; - throw str; - } - file_descriptor.seekg( 0, file_descriptor.end ); - size_t length = size_t( file_descriptor.tellg() ); - file_descriptor.seekg( 0, file_descriptor.beg ); - my_content.resize( length ); - char* begin = &*my_content.begin(); - file_descriptor.read( begin, length ); - file_descriptor.close(); - } - const char* content() { return &*my_content.cbegin(); } - size_t length() { return my_content.length(); } - private: - std::string my_content; - }; - - class opencl_program_builder { - public: - typedef void (CL_CALLBACK *cl_callback_type)(cl_program, void*); - opencl_program_builder( Factory& f, const std::string& name, cl_program program, - cl_uint num_devices, cl_device_id* device_list, - const char* options, cl_callback_type callback, - void* user_data ) { - cl_int err = clBuildProgram( program, num_devices, device_list, options, - callback, user_data ); - if( err == CL_SUCCESS ) - return; - std::string str = std::string( "Failed to build program: " ) + name; - if ( err == CL_BUILD_PROGRAM_FAILURE ) { - const opencl_device_list &devices = f.devices(); - for ( auto d = devices.begin(); d != devices.end(); ++d ) { - std::cerr << "Build log for device: " << (*d).name() << std::endl; - size_t log_size; - cl_int query_err = clGetProgramBuildInfo( - program, (*d).my_cl_device_id, CL_PROGRAM_BUILD_LOG, 0, NULL, - &log_size ); - enforce_cl_retcode( query_err, "Failed to get build log size" ); - if( log_size ) { - std::vector output; - output.resize( log_size ); - query_err = clGetProgramBuildInfo( - program, (*d).my_cl_device_id, CL_PROGRAM_BUILD_LOG, - output.size(), output.data(), NULL ); - enforce_cl_retcode( query_err, "Failed to get build output" ); - std::cerr << output.data() << std::endl; - } else { - std::cerr << "No build log available" << std::endl; - } - } - } - enforce_cl_retcode( err, str ); - } - }; - - class opencl_device_filter { - public: - template - opencl_device_filter( cl_uint& num_devices, cl_device_id* device_list, - Filter filter, const char* message ) { - for ( cl_uint i = 0; i < num_devices; ++i ) - if ( filter(device_list[i]) ) { - device_list[i--] = device_list[--num_devices]; - } - if ( !num_devices ) - enforce_cl_retcode( CL_DEVICE_NOT_AVAILABLE, message ); - } - }; - - void init( const std::string& ) const { - cl_uint num_devices; - enforce_cl_retcode( clGetContextInfo( my_factory.context(), CL_CONTEXT_NUM_DEVICES, sizeof( num_devices ), &num_devices, NULL ), - "Failed to get OpenCL context info" ); - if ( !num_devices ) - enforce_cl_retcode( CL_DEVICE_NOT_FOUND, "No supported devices found" ); - cl_device_id *device_list = (cl_device_id *)alloca( num_devices*sizeof( cl_device_id ) ); - enforce_cl_retcode( clGetContextInfo( my_factory.context(), CL_CONTEXT_DEVICES, num_devices*sizeof( cl_device_id ), device_list, NULL ), - "Failed to get OpenCL context info" ); - const char *options = NULL; - switch ( my_type ) { - case opencl_program_type::SOURCE: { - file_reader fr( my_arg_str ); - const char *s[] = { fr.content() }; - const size_t l[] = { fr.length() }; - cl_int err; - my_cl_program = clCreateProgramWithSource( my_factory.context(), 1, s, l, &err ); - enforce_cl_retcode( err, std::string( "Failed to create program: " ) + my_arg_str ); - opencl_device_filter( - num_devices, device_list, - []( const opencl_device& d ) -> bool { - return !d.compiler_available() || !d.linker_available(); - }, "No one device supports building program from sources" ); - opencl_program_builder( - my_factory, my_arg_str, my_cl_program, num_devices, device_list, - options, /*callback*/ NULL, /*user data*/NULL ); - break; - } - case opencl_program_type::SPIR: - options = "-x spir"; - case opencl_program_type::PRECOMPILED: { - file_reader fr( my_arg_str ); - std::vector s( - num_devices, reinterpret_cast(fr.content()) ); - std::vector l( num_devices, fr.length() ); - std::vector bin_statuses( num_devices, -1 ); - cl_int err; - my_cl_program = clCreateProgramWithBinary( my_factory.context(), num_devices, - device_list, l.data(), s.data(), - bin_statuses.data(), &err ); - if( err != CL_SUCCESS ) { - std::string statuses_str; - for (auto st = bin_statuses.begin(); st != bin_statuses.end(); ++st) { - statuses_str += std::to_string((*st)); - } - - enforce_cl_retcode( err, std::string( "Failed to create program, error " + std::to_string( err ) + " : " ) + my_arg_str + - std::string( ", binary_statuses = " ) + statuses_str ); - } - opencl_program_builder( - my_factory, my_arg_str, my_cl_program, num_devices, device_list, - options, /*callback*/ NULL, /*user data*/NULL ); - break; - } - default: - __TBB_ASSERT( false, "Unsupported program type" ); - } - } - - Factory& my_factory; - opencl_program_type my_type; - std::string my_arg_str; - mutable cl_program my_cl_program; - mutable std::once_flag my_do_once_flag; - - template - friend class opencl_factory; - - template - friend class opencl_factory::kernel; -}; - -template -class opencl_node; - -template -class opencl_node< tuple, JP, Factory > : public streaming_node< tuple, JP, Factory > { - typedef streaming_node < tuple, JP, Factory > base_type; -public: - typedef typename base_type::kernel_type kernel_type; - - opencl_node( graph &g, const kernel_type& kernel ) - : base_type( g, kernel, opencl_info::default_device_selector< opencl_info::default_opencl_factory >(), opencl_info::default_factory() ) - { - tbb::internal::fgt_multiinput_multioutput_node( CODEPTR(), tbb::internal::FLOW_OPENCL_NODE, this, &this->my_graph ); - } - - opencl_node( graph &g, const kernel_type& kernel, Factory &f ) - : base_type( g, kernel, opencl_info::default_device_selector (), f ) - { - tbb::internal::fgt_multiinput_multioutput_node( CODEPTR(), tbb::internal::FLOW_OPENCL_NODE, this, &this->my_graph ); - } - - template - opencl_node( graph &g, const kernel_type& kernel, DeviceSelector d, Factory &f) - : base_type( g, kernel, d, f) - { - tbb::internal::fgt_multiinput_multioutput_node( CODEPTR(), tbb::internal::FLOW_OPENCL_NODE, this, &this->my_graph ); - } -}; - -template -class opencl_node< tuple, JP > : public opencl_node < tuple, JP, opencl_info::default_opencl_factory > { - typedef opencl_node < tuple, JP, opencl_info::default_opencl_factory > base_type; -public: - typedef typename base_type::kernel_type kernel_type; - - opencl_node( graph &g, const kernel_type& kernel ) - : base_type( g, kernel, opencl_info::default_device_selector< opencl_info::default_opencl_factory >(), opencl_info::default_factory() ) - {} - - template - opencl_node( graph &g, const kernel_type& kernel, DeviceSelector d ) - : base_type( g, kernel, d, opencl_info::default_factory() ) - {} -}; - -template -class opencl_node< tuple > : public opencl_node < tuple, queueing, opencl_info::default_opencl_factory > { - typedef opencl_node < tuple, queueing, opencl_info::default_opencl_factory > base_type; -public: - typedef typename base_type::kernel_type kernel_type; - - opencl_node( graph &g, const kernel_type& kernel ) - : base_type( g, kernel, opencl_info::default_device_selector< opencl_info::default_opencl_factory >(), opencl_info::default_factory() ) - {} - - template - opencl_node( graph &g, const kernel_type& kernel, DeviceSelector d ) - : base_type( g, kernel, d, opencl_info::default_factory() ) - {} -}; - -} // namespace interfaceX - -using interface11::opencl_node; -using interface11::read_only; -using interface11::read_write; -using interface11::write_only; -using interface11::opencl_buffer; -using interface11::opencl_subbuffer; -using interface11::opencl_device; -using interface11::opencl_device_list; -using interface11::opencl_program; -using interface11::opencl_program_type; -using interface11::opencl_async_msg; -using interface11::opencl_factory; -using interface11::opencl_range; - -} // namespace flow -} // namespace tbb -#endif /* __TBB_PREVIEW_OPENCL_NODE */ - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_flow_graph_opencl_node_H_include_area - -#endif // __TBB_flow_graph_opencl_node_H diff --git a/src/3rd party/tbb/global_control.h b/src/3rd party/tbb/global_control.h index 505720cba5..2688996ecb 100644 --- a/src/3rd party/tbb/global_control.h +++ b/src/3rd party/tbb/global_control.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,65 +14,4 @@ limitations under the License. */ -#ifndef __TBB_global_control_H -#define __TBB_global_control_H - -#include "tbb_stddef.h" - -namespace tbb { -namespace interface9 { - -class global_control { -public: - enum parameter { - max_allowed_parallelism, - thread_stack_size, - parameter_max // insert new parameters above this point - }; - - global_control(parameter p, size_t value) : - my_value(value), my_next(NULL), my_param(p) { - __TBB_ASSERT(my_param < parameter_max, "Invalid parameter"); -#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) - // For Windows 8 Store* apps it's impossible to set stack size - if (p==thread_stack_size) - return; -#elif __TBB_x86_64 && (_WIN32 || _WIN64) - if (p==thread_stack_size) - __TBB_ASSERT_RELEASE((unsigned)value == value, "Stack size is limited to unsigned int range"); -#endif - if (my_param==max_allowed_parallelism) - __TBB_ASSERT_RELEASE(my_value>0, "max_allowed_parallelism cannot be 0."); - internal_create(); - } - - ~global_control() { - __TBB_ASSERT(my_param < parameter_max, "Invalid parameter. Probably the object was corrupted."); -#if __TBB_WIN8UI_SUPPORT && (_WIN32_WINNT < 0x0A00) - // For Windows 8 Store* apps it's impossible to set stack size - if (my_param==thread_stack_size) - return; -#endif - internal_destroy(); - } - - static size_t active_value(parameter p) { - __TBB_ASSERT(p < parameter_max, "Invalid parameter"); - return active_value((int)p); - } -private: - size_t my_value; - global_control *my_next; - parameter my_param; - - void __TBB_EXPORTED_METHOD internal_create(); - void __TBB_EXPORTED_METHOD internal_destroy(); - static size_t __TBB_EXPORTED_FUNC active_value(int param); -}; -} // namespace interface9 - -using interface9::global_control; - -} // tbb - -#endif // __TBB_global_control_H +#include "../oneapi/tbb/global_control.h" diff --git a/src/3rd party/tbb/index.html b/src/3rd party/tbb/index.html deleted file mode 100644 index 7bbb672d39..0000000000 --- a/src/3rd party/tbb/index.html +++ /dev/null @@ -1,29 +0,0 @@ - - - -

Overview

-Include files for Intel® Threading Building Blocks classes and functions. - -
Click here to see all files in the directory. - -

Directories

-
-
compat -
Include files for source level compatibility with other frameworks. -
internal -
Include files with implementation details; not for direct use. -
machine -
Include files for low-level architecture specific functionality; not for direct use. -
- -
-Up to parent directory -

-Copyright © 2005-2019 Intel Corporation. All Rights Reserved. -

-Intel is a registered trademark or trademark of Intel Corporation -or its subsidiaries in the United States and other countries. -

-* Other names and brands may be claimed as the property of others. - - diff --git a/src/3rd party/tbb/info.h b/src/3rd party/tbb/info.h index 0accc16577..02d331650e 100644 --- a/src/3rd party/tbb/info.h +++ b/src/3rd party/tbb/info.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2019 Intel Corporation + Copyright (c) 2019-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,39 +14,4 @@ limitations under the License. */ -#ifndef __TBB_info_H -#define __TBB_info_H - -#include "tbb_config.h" - -#if __TBB_NUMA_SUPPORT - -#include - -namespace tbb { - namespace internal { - namespace numa_topology { - unsigned nodes_count(); - void fill(int* indexes_array); - int default_concurrency(int node_id); - } //namespace numa_topology - } // namespace internal - - typedef int numa_node_id; - - namespace info { - inline std::vector numa_nodes() { - std::vector nodes_indexes(tbb::internal::numa_topology::nodes_count()); - internal::numa_topology::fill(&nodes_indexes.front()); - return nodes_indexes; - } - - inline int default_concurrency(numa_node_id id = -1) { - return internal::numa_topology::default_concurrency(id); - } - } // namespace info -} // namespace tbb - -#endif /*__TBB_NUMA_SUPPORT*/ - -#endif /*__TBB_info_H*/ +#include "../oneapi/tbb/info.h" diff --git a/src/3rd party/tbb/internal/_aggregator_impl.h b/src/3rd party/tbb/internal/_aggregator_impl.h deleted file mode 100644 index 684e004c90..0000000000 --- a/src/3rd party/tbb/internal/_aggregator_impl.h +++ /dev/null @@ -1,180 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB__aggregator_impl_H -#define __TBB__aggregator_impl_H - -#include "../atomic.h" -#if !__TBBMALLOC_BUILD -#include "../tbb_profiling.h" -#endif - -namespace tbb { -namespace interface6 { -namespace internal { - -using namespace tbb::internal; - -//! aggregated_operation base class -template -class aggregated_operation { - public: - //! Zero value means "wait" status, all other values are "user" specified values and are defined into the scope of a class which uses "status". - uintptr_t status; - - Derived *next; - aggregated_operation() : status(0), next(NULL) {} -}; - -//! Aggregator base class -/** An aggregator for collecting operations coming from multiple sources and executing - them serially on a single thread. operation_type must be derived from - aggregated_operation. The parameter handler_type is a functor that will be passed the - list of operations and is expected to handle each operation appropriately, setting the - status of each operation to non-zero.*/ -template < typename operation_type > -class aggregator_generic { -public: - aggregator_generic() : handler_busy(false) { pending_operations = NULL; } - - //! Execute an operation - /** Places an operation into the waitlist (pending_operations), and either handles the list, - or waits for the operation to complete, or returns. - The long_life_time parameter specifies the life time of the given operation object. - Operations with long_life_time == true may be accessed after execution. - A "short" life time operation (long_life_time == false) can be destroyed - during execution, and so any access to it after it was put into the waitlist, - including status check, is invalid. As a consequence, waiting for completion - of such operation causes undefined behavior. - */ - template < typename handler_type > - void execute(operation_type *op, handler_type &handle_operations, bool long_life_time = true) { - operation_type *res; - // op->status should be read before inserting the operation into the - // aggregator waitlist since it can become invalid after executing a - // handler (if the operation has 'short' life time.) - const uintptr_t status = op->status; - - // ITT note: &(op->status) tag is used to cover accesses to this op node. This - // thread has created the operation, and now releases it so that the handler - // thread may handle the associated operation w/o triggering a race condition; - // thus this tag will be acquired just before the operation is handled in the - // handle_operations functor. - call_itt_notify(releasing, &(op->status)); - // insert the operation in the queue. - do { - // Tools may flag the following line as a race; it is a false positive: - // This is an atomic read; we don't provide itt_hide_load_word for atomics - op->next = res = pending_operations; // NOT A RACE - } while (pending_operations.compare_and_swap(op, res) != res); - if (!res) { // first in the list; handle the operations. - // ITT note: &pending_operations tag covers access to the handler_busy flag, - // which this waiting handler thread will try to set before entering - // handle_operations. - call_itt_notify(acquired, &pending_operations); - start_handle_operations(handle_operations); - // The operation with 'short' life time can already be destroyed. - if (long_life_time) - __TBB_ASSERT(op->status, NULL); - } - // not first; wait for op to be ready. - else if (!status) { // operation is blocking here. - __TBB_ASSERT(long_life_time, "Waiting for an operation object that might be destroyed during processing."); - call_itt_notify(prepare, &(op->status)); - spin_wait_while_eq(op->status, uintptr_t(0)); - itt_load_word_with_acquire(op->status); - } - } - - private: - //! An atomically updated list (aka mailbox) of pending operations - atomic pending_operations; - //! Controls thread access to handle_operations - uintptr_t handler_busy; - - //! Trigger the handling of operations when the handler is free - template < typename handler_type > - void start_handle_operations( handler_type &handle_operations ) { - operation_type *op_list; - - // ITT note: &handler_busy tag covers access to pending_operations as it is passed - // between active and waiting handlers. Below, the waiting handler waits until - // the active handler releases, and the waiting handler acquires &handler_busy as - // it becomes the active_handler. The release point is at the end of this - // function, when all operations in pending_operations have been handled by the - // owner of this aggregator. - call_itt_notify(prepare, &handler_busy); - // get the handler_busy: - // only one thread can possibly spin here at a time - spin_wait_until_eq(handler_busy, uintptr_t(0)); - call_itt_notify(acquired, &handler_busy); - // acquire fence not necessary here due to causality rule and surrounding atomics - __TBB_store_with_release(handler_busy, uintptr_t(1)); - - // ITT note: &pending_operations tag covers access to the handler_busy flag - // itself. Capturing the state of the pending_operations signifies that - // handler_busy has been set and a new active handler will now process that list's - // operations. - call_itt_notify(releasing, &pending_operations); - // grab pending_operations - op_list = pending_operations.fetch_and_store(NULL); - - // handle all the operations - handle_operations(op_list); - - // release the handler - itt_store_word_with_release(handler_busy, uintptr_t(0)); - } -}; - -template < typename handler_type, typename operation_type > -class aggregator : public aggregator_generic { - handler_type handle_operations; -public: - aggregator() {} - explicit aggregator(handler_type h) : handle_operations(h) {} - - void initialize_handler(handler_type h) { handle_operations = h; } - - void execute(operation_type *op) { - aggregator_generic::execute(op, handle_operations); - } -}; - -// the most-compatible friend declaration (vs, gcc, icc) is -// template friend class aggregating_functor; -template -class aggregating_functor { - aggregating_class *fi; -public: - aggregating_functor() : fi() {} - aggregating_functor(aggregating_class *fi_) : fi(fi_) {} - void operator()(operation_list* op_list) { fi->handle_operations(op_list); } -}; - -} // namespace internal -} // namespace interface6 - -namespace internal { - using interface6::internal::aggregated_operation; - using interface6::internal::aggregator_generic; - using interface6::internal::aggregator; - using interface6::internal::aggregating_functor; -} // namespace internal - -} // namespace tbb - -#endif // __TBB__aggregator_impl_H diff --git a/src/3rd party/tbb/internal/_allocator_traits.h b/src/3rd party/tbb/internal/_allocator_traits.h deleted file mode 100644 index 272077a6a9..0000000000 --- a/src/3rd party/tbb/internal/_allocator_traits.h +++ /dev/null @@ -1,156 +0,0 @@ -/* - Copyright (c) 2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_allocator_traits_H -#define __TBB_allocator_traits_H - -#include "../tbb_stddef.h" // true/false_type - -#if __TBB_ALLOCATOR_TRAITS_PRESENT -#include // for allocator_traits -#endif - -#if __TBB_CPP11_RVALUE_REF_PRESENT -#include // for std::move -#endif - -// For allocator_swap helper -#include __TBB_STD_SWAP_HEADER - -namespace tbb { -namespace internal { - -//! Internal implementation of allocator traits, propagate_on_* use internal boolean_constant. -//! In order to avoid code duplication, check what implementation of boolean constant will likely be passed. -#if __TBB_ALLOCATOR_TRAITS_PRESENT -typedef std::true_type traits_true_type; -typedef std::false_type traits_false_type; -#else -typedef tbb::internal::true_type traits_true_type; -typedef tbb::internal::false_type traits_false_type; -#endif - -//! Copy assignment implementation for allocator if propagate_on_container_copy_assignment == true_type -//! Noop if pocca == false_type -template -inline void allocator_copy_assignment(MyAlloc& my_allocator, OtherAlloc& other_allocator, traits_true_type) { - my_allocator = other_allocator; -} -template -inline void allocator_copy_assignment(MyAlloc&, OtherAlloc&, traits_false_type) { /* NO COPY */} - -#if __TBB_CPP11_RVALUE_REF_PRESENT -//! Move assignment implementation for allocator if propagate_on_container_move_assignment == true_type. -//! Noop if pocma == false_type. -template -inline void allocator_move_assignment(MyAlloc& my_allocator, OtherAlloc& other_allocator, traits_true_type) { - my_allocator = std::move(other_allocator); -} -template -inline void allocator_move_assignment(MyAlloc&, OtherAlloc&, traits_false_type) { /* NO MOVE */ } -#endif - -//! Swap implementation for allocators if propagate_on_container_swap == true_type. -//! Noop if pocs == false_type. -template -inline void allocator_swap(MyAlloc& my_allocator, OtherAlloc& other_allocator, traits_true_type) { - using std::swap; - swap(my_allocator, other_allocator); -} -template -inline void allocator_swap(MyAlloc&, OtherAlloc&, traits_false_type) { /* NO SWAP */ } - -#if __TBB_ALLOCATOR_TRAITS_PRESENT -using std::allocator_traits; -#else -//! Internal allocator_traits implementation, which relies on C++03 standard -//! [20.1.5] allocator requirements -template -struct allocator_traits { - // C++03 allocator doesn't have to be assignable or swappable, therefore - // define these traits as false_type to do not require additional operations - // that are not supposed to be in. - typedef tbb::internal::false_type propagate_on_container_move_assignment; - typedef tbb::internal::false_type propagate_on_container_copy_assignment; - typedef tbb::internal::false_type propagate_on_container_swap; - - typedef Alloc allocator_type; - typedef typename allocator_type::value_type value_type; - - typedef typename allocator_type::pointer pointer; - typedef typename allocator_type::const_pointer const_pointer; - typedef typename allocator_type::difference_type difference_type; - typedef typename allocator_type::size_type size_type; - - template struct rebind_alloc { - typedef typename Alloc::template rebind::other other; - }; - - static pointer allocate(Alloc& a, size_type n) { - return a.allocate(n); - } - - static void deallocate(Alloc& a, pointer p, size_type n) { - a.deallocate(p, n); - } - - template - static void construct(Alloc&, PT* p) { - ::new (static_cast(p)) PT(); - } - - template - static void construct(Alloc&, PT* p, __TBB_FORWARDING_REF(T1) t1) { - ::new (static_cast(p)) PT(tbb::internal::forward(t1)); - } - - template - static void construct(Alloc&, PT* p, __TBB_FORWARDING_REF(T1) t1, __TBB_FORWARDING_REF(T2) t2) { - ::new (static_cast(p)) PT(tbb::internal::forward(t1), tbb::internal::forward(t2)); - } - - template - static void construct(Alloc&, PT* p, __TBB_FORWARDING_REF(T1) t1, - __TBB_FORWARDING_REF(T2) t2, __TBB_FORWARDING_REF(T3) t3) { - ::new (static_cast(p)) PT(tbb::internal::forward(t1), tbb::internal::forward(t2), - tbb::internal::forward(t3)); - } - - template - static void destroy(Alloc&, T* p) { - p->~T(); - tbb::internal::suppress_unused_warning(p); - } - - static Alloc select_on_container_copy_construction(const Alloc& a) { return a; } -}; -#endif // __TBB_ALLOCATOR_TRAITS_PRESENT - -//! C++03/C++11 compliant rebind helper, even if no std::allocator_traits available -//! or rebind is not defined for allocator type -template -struct allocator_rebind { -#if __TBB_ALLOCATOR_TRAITS_PRESENT - typedef typename allocator_traits::template rebind_alloc type; -#else - typedef typename allocator_traits::template rebind_alloc::other type; -#endif -}; - -}} // namespace tbb::internal - -#endif // __TBB_allocator_traits_H - diff --git a/src/3rd party/tbb/internal/_concurrent_queue_impl.h b/src/3rd party/tbb/internal/_concurrent_queue_impl.h deleted file mode 100644 index 594f952f63..0000000000 --- a/src/3rd party/tbb/internal/_concurrent_queue_impl.h +++ /dev/null @@ -1,1081 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB__concurrent_queue_impl_H -#define __TBB__concurrent_queue_impl_H - -#ifndef __TBB_concurrent_queue_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include "../tbb_stddef.h" -#include "../tbb_machine.h" -#include "../atomic.h" -#include "../spin_mutex.h" -#include "../cache_aligned_allocator.h" -#include "../tbb_exception.h" -#include "../tbb_profiling.h" -#include -#include __TBB_STD_SWAP_HEADER -#include - -namespace tbb { - -#if !__TBB_TEMPLATE_FRIENDS_BROKEN - -// forward declaration -namespace strict_ppl { -template class concurrent_queue; -} - -template class concurrent_bounded_queue; - -#endif - -//! For internal use only. -namespace strict_ppl { - -//! @cond INTERNAL -namespace internal { - -using namespace tbb::internal; - -typedef size_t ticket; - -template class micro_queue ; -template class micro_queue_pop_finalizer ; -template class concurrent_queue_base_v3; -template struct concurrent_queue_rep; - -//! parts of concurrent_queue_rep that do not have references to micro_queue -/** - * For internal use only. - */ -struct concurrent_queue_rep_base : no_copy { - template friend class micro_queue; - template friend class concurrent_queue_base_v3; - -protected: - //! Approximately n_queue/golden ratio - static const size_t phi = 3; - -public: - // must be power of 2 - static const size_t n_queue = 8; - - //! Prefix on a page - struct page { - page* next; - uintptr_t mask; - }; - - atomic head_counter; - char pad1[NFS_MaxLineSize-sizeof(atomic)]; - atomic tail_counter; - char pad2[NFS_MaxLineSize-sizeof(atomic)]; - - //! Always a power of 2 - size_t items_per_page; - - //! Size of an item - size_t item_size; - - //! number of invalid entries in the queue - atomic n_invalid_entries; - - char pad3[NFS_MaxLineSize-sizeof(size_t)-sizeof(size_t)-sizeof(atomic)]; -} ; - -inline bool is_valid_page(const concurrent_queue_rep_base::page* p) { - return uintptr_t(p)>1; -} - -//! Abstract class to define interface for page allocation/deallocation -/** - * For internal use only. - */ -class concurrent_queue_page_allocator -{ - template friend class micro_queue ; - template friend class micro_queue_pop_finalizer ; -protected: - virtual ~concurrent_queue_page_allocator() {} -private: - virtual concurrent_queue_rep_base::page* allocate_page() = 0; - virtual void deallocate_page( concurrent_queue_rep_base::page* p ) = 0; -} ; - -#if _MSC_VER && !defined(__INTEL_COMPILER) -// unary minus operator applied to unsigned type, result still unsigned -#pragma warning( push ) -#pragma warning( disable: 4146 ) -#endif - -//! A queue using simple locking. -/** For efficiency, this class has no constructor. - The caller is expected to zero-initialize it. */ -template -class micro_queue : no_copy { -public: - typedef void (*item_constructor_t)(T* location, const void* src); -private: - typedef concurrent_queue_rep_base::page page; - - //! Class used to ensure exception-safety of method "pop" - class destroyer: no_copy { - T& my_value; - public: - destroyer( T& value ) : my_value(value) {} - ~destroyer() {my_value.~T();} - }; - - void copy_item( page& dst, size_t dindex, const void* src, item_constructor_t construct_item ) { - construct_item( &get_ref(dst, dindex), src ); - } - - void copy_item( page& dst, size_t dindex, const page& src, size_t sindex, - item_constructor_t construct_item ) - { - T& src_item = get_ref( const_cast(src), sindex ); - construct_item( &get_ref(dst, dindex), static_cast(&src_item) ); - } - - void assign_and_destroy_item( void* dst, page& src, size_t index ) { - T& from = get_ref(src,index); - destroyer d(from); - *static_cast(dst) = tbb::internal::move( from ); - } - - void spin_wait_until_my_turn( atomic& counter, ticket k, concurrent_queue_rep_base& rb ) const ; - -public: - friend class micro_queue_pop_finalizer; - - struct padded_page: page { - //! Not defined anywhere - exists to quiet warnings. - padded_page(); - //! Not defined anywhere - exists to quiet warnings. - void operator=( const padded_page& ); - //! Must be last field. - T last; - }; - - static T& get_ref( page& p, size_t index ) { - return (&static_cast(static_cast(&p))->last)[index]; - } - - atomic head_page; - atomic head_counter; - - atomic tail_page; - atomic tail_counter; - - spin_mutex page_mutex; - - void push( const void* item, ticket k, concurrent_queue_base_v3& base, - item_constructor_t construct_item ) ; - - bool pop( void* dst, ticket k, concurrent_queue_base_v3& base ) ; - - micro_queue& assign( const micro_queue& src, concurrent_queue_base_v3& base, - item_constructor_t construct_item ) ; - - page* make_copy( concurrent_queue_base_v3& base, const page* src_page, size_t begin_in_page, - size_t end_in_page, ticket& g_index, item_constructor_t construct_item ) ; - - void invalidate_page_and_rethrow( ticket k ) ; -}; - -template -void micro_queue::spin_wait_until_my_turn( atomic& counter, ticket k, concurrent_queue_rep_base& rb ) const { - for( atomic_backoff b(true);;b.pause() ) { - ticket c = counter; - if( c==k ) return; - else if( c&1 ) { - ++rb.n_invalid_entries; - throw_exception( eid_bad_last_alloc ); - } - } -} - -template -void micro_queue::push( const void* item, ticket k, concurrent_queue_base_v3& base, - item_constructor_t construct_item ) -{ - k &= -concurrent_queue_rep_base::n_queue; - page* p = NULL; - size_t index = modulo_power_of_two( k/concurrent_queue_rep_base::n_queue, base.my_rep->items_per_page); - if( !index ) { - __TBB_TRY { - concurrent_queue_page_allocator& pa = base; - p = pa.allocate_page(); - } __TBB_CATCH (...) { - ++base.my_rep->n_invalid_entries; - invalidate_page_and_rethrow( k ); - } - p->mask = 0; - p->next = NULL; - } - - if( tail_counter != k ) spin_wait_until_my_turn( tail_counter, k, *base.my_rep ); - call_itt_notify(acquired, &tail_counter); - - if( p ) { - spin_mutex::scoped_lock lock( page_mutex ); - page* q = tail_page; - if( is_valid_page(q) ) - q->next = p; - else - head_page = p; - tail_page = p; - } else { - p = tail_page; - } - - __TBB_TRY { - copy_item( *p, index, item, construct_item ); - // If no exception was thrown, mark item as present. - itt_hide_store_word(p->mask, p->mask | uintptr_t(1)<n_invalid_entries; - call_itt_notify(releasing, &tail_counter); - tail_counter += concurrent_queue_rep_base::n_queue; - __TBB_RETHROW(); - } -} - -template -bool micro_queue::pop( void* dst, ticket k, concurrent_queue_base_v3& base ) { - k &= -concurrent_queue_rep_base::n_queue; - if( head_counter!=k ) spin_wait_until_eq( head_counter, k ); - call_itt_notify(acquired, &head_counter); - if( tail_counter==k ) spin_wait_while_eq( tail_counter, k ); - call_itt_notify(acquired, &tail_counter); - page *p = head_page; - __TBB_ASSERT( p, NULL ); - size_t index = modulo_power_of_two( k/concurrent_queue_rep_base::n_queue, base.my_rep->items_per_page ); - bool success = false; - { - micro_queue_pop_finalizer finalizer( *this, base, k+concurrent_queue_rep_base::n_queue, index==base.my_rep->items_per_page-1 ? p : NULL ); - if( p->mask & uintptr_t(1)<n_invalid_entries; - } - } - return success; -} - -template -micro_queue& micro_queue::assign( const micro_queue& src, concurrent_queue_base_v3& base, - item_constructor_t construct_item ) -{ - head_counter = src.head_counter; - tail_counter = src.tail_counter; - - const page* srcp = src.head_page; - if( is_valid_page(srcp) ) { - ticket g_index = head_counter; - __TBB_TRY { - size_t n_items = (tail_counter-head_counter)/concurrent_queue_rep_base::n_queue; - size_t index = modulo_power_of_two( head_counter/concurrent_queue_rep_base::n_queue, base.my_rep->items_per_page ); - size_t end_in_first_page = (index+n_itemsitems_per_page)?(index+n_items):base.my_rep->items_per_page; - - head_page = make_copy( base, srcp, index, end_in_first_page, g_index, construct_item ); - page* cur_page = head_page; - - if( srcp != src.tail_page ) { - for( srcp = srcp->next; srcp!=src.tail_page; srcp=srcp->next ) { - cur_page->next = make_copy( base, srcp, 0, base.my_rep->items_per_page, g_index, construct_item ); - cur_page = cur_page->next; - } - - __TBB_ASSERT( srcp==src.tail_page, NULL ); - size_t last_index = modulo_power_of_two( tail_counter/concurrent_queue_rep_base::n_queue, base.my_rep->items_per_page ); - if( last_index==0 ) last_index = base.my_rep->items_per_page; - - cur_page->next = make_copy( base, srcp, 0, last_index, g_index, construct_item ); - cur_page = cur_page->next; - } - tail_page = cur_page; - } __TBB_CATCH (...) { - invalidate_page_and_rethrow( g_index ); - } - } else { - head_page = tail_page = NULL; - } - return *this; -} - -template -void micro_queue::invalidate_page_and_rethrow( ticket k ) { - // Append an invalid page at address 1 so that no more pushes are allowed. - page* invalid_page = (page*)uintptr_t(1); - { - spin_mutex::scoped_lock lock( page_mutex ); - itt_store_word_with_release(tail_counter, k+concurrent_queue_rep_base::n_queue+1); - page* q = tail_page; - if( is_valid_page(q) ) - q->next = invalid_page; - else - head_page = invalid_page; - tail_page = invalid_page; - } - __TBB_RETHROW(); -} - -template -concurrent_queue_rep_base::page* micro_queue::make_copy( concurrent_queue_base_v3& base, - const concurrent_queue_rep_base::page* src_page, size_t begin_in_page, size_t end_in_page, - ticket& g_index, item_constructor_t construct_item ) -{ - concurrent_queue_page_allocator& pa = base; - page* new_page = pa.allocate_page(); - new_page->next = NULL; - new_page->mask = src_page->mask; - for( ; begin_in_page!=end_in_page; ++begin_in_page, ++g_index ) - if( new_page->mask & uintptr_t(1)< -class micro_queue_pop_finalizer: no_copy { - typedef concurrent_queue_rep_base::page page; - ticket my_ticket; - micro_queue& my_queue; - page* my_page; - concurrent_queue_page_allocator& allocator; -public: - micro_queue_pop_finalizer( micro_queue& queue, concurrent_queue_base_v3& b, ticket k, page* p ) : - my_ticket(k), my_queue(queue), my_page(p), allocator(b) - {} - ~micro_queue_pop_finalizer() ; -}; - -template -micro_queue_pop_finalizer::~micro_queue_pop_finalizer() { - page* p = my_page; - if( is_valid_page(p) ) { - spin_mutex::scoped_lock lock( my_queue.page_mutex ); - page* q = p->next; - my_queue.head_page = q; - if( !is_valid_page(q) ) { - my_queue.tail_page = NULL; - } - } - itt_store_word_with_release(my_queue.head_counter, my_ticket); - if( is_valid_page(p) ) { - allocator.deallocate_page( p ); - } -} - -#if _MSC_VER && !defined(__INTEL_COMPILER) -#pragma warning( pop ) -#endif // warning 4146 is back - -template class concurrent_queue_iterator_rep ; -template class concurrent_queue_iterator_base_v3; - -//! representation of concurrent_queue_base -/** - * the class inherits from concurrent_queue_rep_base and defines an array of micro_queue's - */ -template -struct concurrent_queue_rep : public concurrent_queue_rep_base { - micro_queue array[n_queue]; - - //! Map ticket to an array index - static size_t index( ticket k ) { - return k*phi%n_queue; - } - - micro_queue& choose( ticket k ) { - // The formula here approximates LRU in a cache-oblivious way. - return array[index(k)]; - } -}; - -//! base class of concurrent_queue -/** - * The class implements the interface defined by concurrent_queue_page_allocator - * and has a pointer to an instance of concurrent_queue_rep. - */ -template -class concurrent_queue_base_v3: public concurrent_queue_page_allocator { -private: - //! Internal representation - concurrent_queue_rep* my_rep; - - friend struct concurrent_queue_rep; - friend class micro_queue; - friend class concurrent_queue_iterator_rep; - friend class concurrent_queue_iterator_base_v3; - -protected: - typedef typename concurrent_queue_rep::page page; - -private: - typedef typename micro_queue::padded_page padded_page; - typedef typename micro_queue::item_constructor_t item_constructor_t; - - virtual page *allocate_page() __TBB_override { - concurrent_queue_rep& r = *my_rep; - size_t n = sizeof(padded_page) + (r.items_per_page-1)*sizeof(T); - return reinterpret_cast(allocate_block ( n )); - } - - virtual void deallocate_page( concurrent_queue_rep_base::page *p ) __TBB_override { - concurrent_queue_rep& r = *my_rep; - size_t n = sizeof(padded_page) + (r.items_per_page-1)*sizeof(T); - deallocate_block( reinterpret_cast(p), n ); - } - - //! custom allocator - virtual void *allocate_block( size_t n ) = 0; - - //! custom de-allocator - virtual void deallocate_block( void *p, size_t n ) = 0; - -protected: - concurrent_queue_base_v3(); - - virtual ~concurrent_queue_base_v3() { -#if TBB_USE_ASSERT - size_t nq = my_rep->n_queue; - for( size_t i=0; iarray[i].tail_page==NULL, "pages were not freed properly" ); -#endif /* TBB_USE_ASSERT */ - cache_aligned_allocator >().deallocate(my_rep,1); - } - - //! Enqueue item at tail of queue - void internal_push( const void* src, item_constructor_t construct_item ) { - concurrent_queue_rep& r = *my_rep; - ticket k = r.tail_counter++; - r.choose(k).push( src, k, *this, construct_item ); - } - - //! Attempt to dequeue item from queue. - /** NULL if there was no item to dequeue. */ - bool internal_try_pop( void* dst ) ; - - //! Get size of queue; result may be invalid if queue is modified concurrently - size_t internal_size() const ; - - //! check if the queue is empty; thread safe - bool internal_empty() const ; - - //! free any remaining pages - /* note that the name may be misleading, but it remains so due to a historical accident. */ - void internal_finish_clear() ; - - //! Obsolete - void internal_throw_exception() const { - throw_exception( eid_bad_alloc ); - } - - //! copy or move internal representation - void assign( const concurrent_queue_base_v3& src, item_constructor_t construct_item ) ; - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! swap internal representation - void internal_swap( concurrent_queue_base_v3& src ) { - std::swap( my_rep, src.my_rep ); - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ -}; - -template -concurrent_queue_base_v3::concurrent_queue_base_v3() { - const size_t item_size = sizeof(T); - my_rep = cache_aligned_allocator >().allocate(1); - __TBB_ASSERT( (size_t)my_rep % NFS_GetLineSize()==0, "alignment error" ); - __TBB_ASSERT( (size_t)&my_rep->head_counter % NFS_GetLineSize()==0, "alignment error" ); - __TBB_ASSERT( (size_t)&my_rep->tail_counter % NFS_GetLineSize()==0, "alignment error" ); - __TBB_ASSERT( (size_t)&my_rep->array % NFS_GetLineSize()==0, "alignment error" ); - memset(static_cast(my_rep),0,sizeof(concurrent_queue_rep)); - my_rep->item_size = item_size; - my_rep->items_per_page = item_size<= 8 ? 32 : - item_size<= 16 ? 16 : - item_size<= 32 ? 8 : - item_size<= 64 ? 4 : - item_size<=128 ? 2 : - 1; -} - -template -bool concurrent_queue_base_v3::internal_try_pop( void* dst ) { - concurrent_queue_rep& r = *my_rep; - ticket k; - do { - k = r.head_counter; - for(;;) { - if( (ptrdiff_t)(r.tail_counter-k)<=0 ) { - // Queue is empty - return false; - } - // Queue had item with ticket k when we looked. Attempt to get that item. - ticket tk=k; -#if defined(_MSC_VER) && defined(_Wp64) - #pragma warning (push) - #pragma warning (disable: 4267) -#endif - k = r.head_counter.compare_and_swap( tk+1, tk ); -#if defined(_MSC_VER) && defined(_Wp64) - #pragma warning (pop) -#endif - if( k==tk ) - break; - // Another thread snatched the item, retry. - } - } while( !r.choose( k ).pop( dst, k, *this ) ); - return true; -} - -template -size_t concurrent_queue_base_v3::internal_size() const { - concurrent_queue_rep& r = *my_rep; - __TBB_ASSERT( sizeof(ptrdiff_t)<=sizeof(size_t), NULL ); - ticket hc = r.head_counter; - size_t nie = r.n_invalid_entries; - ticket tc = r.tail_counter; - __TBB_ASSERT( hc!=tc || !nie, NULL ); - ptrdiff_t sz = tc-hc-nie; - return sz<0 ? 0 : size_t(sz); -} - -template -bool concurrent_queue_base_v3::internal_empty() const { - concurrent_queue_rep& r = *my_rep; - ticket tc = r.tail_counter; - ticket hc = r.head_counter; - // if tc!=r.tail_counter, the queue was not empty at some point between the two reads. - return tc==r.tail_counter && tc==hc+r.n_invalid_entries ; -} - -template -void concurrent_queue_base_v3::internal_finish_clear() { - concurrent_queue_rep& r = *my_rep; - size_t nq = r.n_queue; - for( size_t i=0; i -void concurrent_queue_base_v3::assign( const concurrent_queue_base_v3& src, - item_constructor_t construct_item ) -{ - concurrent_queue_rep& r = *my_rep; - r.items_per_page = src.my_rep->items_per_page; - - // copy concurrent_queue_rep data - r.head_counter = src.my_rep->head_counter; - r.tail_counter = src.my_rep->tail_counter; - r.n_invalid_entries = src.my_rep->n_invalid_entries; - - // copy or move micro_queues - for( size_t i = 0; i < r.n_queue; ++i ) - r.array[i].assign( src.my_rep->array[i], *this, construct_item); - - __TBB_ASSERT( r.head_counter==src.my_rep->head_counter && r.tail_counter==src.my_rep->tail_counter, - "the source concurrent queue should not be concurrently modified." ); -} - -template class concurrent_queue_iterator; - -template -class concurrent_queue_iterator_rep: no_assign { - typedef typename micro_queue::padded_page padded_page; -public: - ticket head_counter; - const concurrent_queue_base_v3& my_queue; - typename concurrent_queue_base_v3::page* array[concurrent_queue_rep::n_queue]; - concurrent_queue_iterator_rep( const concurrent_queue_base_v3& queue ) : - head_counter(queue.my_rep->head_counter), - my_queue(queue) - { - for( size_t k=0; k::n_queue; ++k ) - array[k] = queue.my_rep->array[k].head_page; - } - - //! Set item to point to kth element. Return true if at end of queue or item is marked valid; false otherwise. - bool get_item( T*& item, size_t k ) ; -}; - -template -bool concurrent_queue_iterator_rep::get_item( T*& item, size_t k ) { - if( k==my_queue.my_rep->tail_counter ) { - item = NULL; - return true; - } else { - typename concurrent_queue_base_v3::page* p = array[concurrent_queue_rep::index(k)]; - __TBB_ASSERT(p,NULL); - size_t i = modulo_power_of_two( k/concurrent_queue_rep::n_queue, my_queue.my_rep->items_per_page ); - item = µ_queue::get_ref(*p,i); - return (p->mask & uintptr_t(1)< -class concurrent_queue_iterator_base_v3 { - //! Represents concurrent_queue over which we are iterating. - /** NULL if one past last element in queue. */ - concurrent_queue_iterator_rep* my_rep; - - template - friend bool operator==( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ); - - template - friend bool operator!=( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ); -protected: - //! Pointer to current item - Value* my_item; - - //! Default constructor - concurrent_queue_iterator_base_v3() : my_rep(NULL), my_item(NULL) { -#if __TBB_GCC_OPTIMIZER_ORDERING_BROKEN - __TBB_compiler_fence(); -#endif - } - - //! Copy constructor - concurrent_queue_iterator_base_v3( const concurrent_queue_iterator_base_v3& i ) - : my_rep(NULL), my_item(NULL) { - assign(i); - } - - concurrent_queue_iterator_base_v3& operator=( const concurrent_queue_iterator_base_v3& i ) { - assign(i); - return *this; - } - - //! Construct iterator pointing to head of queue. - concurrent_queue_iterator_base_v3( const concurrent_queue_base_v3& queue ) ; - - //! Assignment - void assign( const concurrent_queue_iterator_base_v3& other ) ; - - //! Advance iterator one step towards tail of queue. - void advance() ; - - //! Destructor - ~concurrent_queue_iterator_base_v3() { - cache_aligned_allocator >().deallocate(my_rep, 1); - my_rep = NULL; - } -}; - -template -concurrent_queue_iterator_base_v3::concurrent_queue_iterator_base_v3( const concurrent_queue_base_v3& queue ) { - my_rep = cache_aligned_allocator >().allocate(1); - new( my_rep ) concurrent_queue_iterator_rep(queue); - size_t k = my_rep->head_counter; - if( !my_rep->get_item(my_item, k) ) advance(); -} - -template -void concurrent_queue_iterator_base_v3::assign( const concurrent_queue_iterator_base_v3& other ) { - if( my_rep!=other.my_rep ) { - if( my_rep ) { - cache_aligned_allocator >().deallocate(my_rep, 1); - my_rep = NULL; - } - if( other.my_rep ) { - my_rep = cache_aligned_allocator >().allocate(1); - new( my_rep ) concurrent_queue_iterator_rep( *other.my_rep ); - } - } - my_item = other.my_item; -} - -template -void concurrent_queue_iterator_base_v3::advance() { - __TBB_ASSERT( my_item, "attempt to increment iterator past end of queue" ); - size_t k = my_rep->head_counter; - const concurrent_queue_base_v3& queue = my_rep->my_queue; -#if TBB_USE_ASSERT - Value* tmp; - my_rep->get_item(tmp,k); - __TBB_ASSERT( my_item==tmp, NULL ); -#endif /* TBB_USE_ASSERT */ - size_t i = modulo_power_of_two( k/concurrent_queue_rep::n_queue, queue.my_rep->items_per_page ); - if( i==queue.my_rep->items_per_page-1 ) { - typename concurrent_queue_base_v3::page*& root = my_rep->array[concurrent_queue_rep::index(k)]; - root = root->next; - } - // advance k - my_rep->head_counter = ++k; - if( !my_rep->get_item(my_item, k) ) advance(); -} - -//! Similar to C++0x std::remove_cv -/** "tbb_" prefix added to avoid overload confusion with C++0x implementations. */ -template struct tbb_remove_cv {typedef T type;}; -template struct tbb_remove_cv {typedef T type;}; -template struct tbb_remove_cv {typedef T type;}; -template struct tbb_remove_cv {typedef T type;}; - -//! Meets requirements of a forward iterator for STL. -/** Value is either the T or const T type of the container. - @ingroup containers */ -template -class concurrent_queue_iterator: public concurrent_queue_iterator_base_v3::type>, - public std::iterator { -#if !__TBB_TEMPLATE_FRIENDS_BROKEN - template - friend class ::tbb::strict_ppl::concurrent_queue; -#else -public: -#endif - //! Construct iterator pointing to head of queue. - explicit concurrent_queue_iterator( const concurrent_queue_base_v3::type>& queue ) : - concurrent_queue_iterator_base_v3::type>(queue) - { - } - -public: - concurrent_queue_iterator() {} - - /** If Value==Container::value_type, then this routine is the copy constructor. - If Value==const Container::value_type, then this routine is a conversion constructor. */ - concurrent_queue_iterator( const concurrent_queue_iterator& other ) : - concurrent_queue_iterator_base_v3::type>(other) - {} - - //! Iterator assignment - concurrent_queue_iterator& operator=( const concurrent_queue_iterator& other ) { - concurrent_queue_iterator_base_v3::type>::operator=(other); - return *this; - } - - //! Reference to current item - Value& operator*() const { - return *static_cast(this->my_item); - } - - Value* operator->() const {return &operator*();} - - //! Advance to next item in queue - concurrent_queue_iterator& operator++() { - this->advance(); - return *this; - } - - //! Post increment - Value* operator++(int) { - Value* result = &operator*(); - operator++(); - return result; - } -}; // concurrent_queue_iterator - - -template -bool operator==( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ) { - return i.my_item==j.my_item; -} - -template -bool operator!=( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ) { - return i.my_item!=j.my_item; -} - -} // namespace internal - -//! @endcond - -} // namespace strict_ppl - -//! @cond INTERNAL -namespace internal { - -class concurrent_queue_rep; -class concurrent_queue_iterator_rep; -class concurrent_queue_iterator_base_v3; -template class concurrent_queue_iterator; - -//! For internal use only. -/** Type-independent portion of concurrent_queue. - @ingroup containers */ -class concurrent_queue_base_v3: no_copy { -private: - //! Internal representation - concurrent_queue_rep* my_rep; - - friend class concurrent_queue_rep; - friend struct micro_queue; - friend class micro_queue_pop_finalizer; - friend class concurrent_queue_iterator_rep; - friend class concurrent_queue_iterator_base_v3; -protected: - //! Prefix on a page - struct page { - page* next; - uintptr_t mask; - }; - - //! Capacity of the queue - ptrdiff_t my_capacity; - - //! Always a power of 2 - size_t items_per_page; - - //! Size of an item - size_t item_size; - - enum copy_specifics { copy, move }; - -#if __TBB_PROTECTED_NESTED_CLASS_BROKEN -public: -#endif - template - struct padded_page: page { - //! Not defined anywhere - exists to quiet warnings. - padded_page(); - //! Not defined anywhere - exists to quiet warnings. - void operator=( const padded_page& ); - //! Must be last field. - T last; - }; - -private: - virtual void copy_item( page& dst, size_t index, const void* src ) = 0; - virtual void assign_and_destroy_item( void* dst, page& src, size_t index ) = 0; -protected: - __TBB_EXPORTED_METHOD concurrent_queue_base_v3( size_t item_size ); - virtual __TBB_EXPORTED_METHOD ~concurrent_queue_base_v3(); - - //! Enqueue item at tail of queue using copy operation - void __TBB_EXPORTED_METHOD internal_push( const void* src ); - - //! Dequeue item from head of queue - void __TBB_EXPORTED_METHOD internal_pop( void* dst ); - - //! Abort all pending queue operations - void __TBB_EXPORTED_METHOD internal_abort(); - - //! Attempt to enqueue item onto queue using copy operation - bool __TBB_EXPORTED_METHOD internal_push_if_not_full( const void* src ); - - //! Attempt to dequeue item from queue. - /** NULL if there was no item to dequeue. */ - bool __TBB_EXPORTED_METHOD internal_pop_if_present( void* dst ); - - //! Get size of queue - ptrdiff_t __TBB_EXPORTED_METHOD internal_size() const; - - //! Check if the queue is empty - bool __TBB_EXPORTED_METHOD internal_empty() const; - - //! Set the queue capacity - void __TBB_EXPORTED_METHOD internal_set_capacity( ptrdiff_t capacity, size_t element_size ); - - //! custom allocator - virtual page *allocate_page() = 0; - - //! custom de-allocator - virtual void deallocate_page( page *p ) = 0; - - //! free any remaining pages - /* note that the name may be misleading, but it remains so due to a historical accident. */ - void __TBB_EXPORTED_METHOD internal_finish_clear() ; - - //! throw an exception - void __TBB_EXPORTED_METHOD internal_throw_exception() const; - - //! copy internal representation - void __TBB_EXPORTED_METHOD assign( const concurrent_queue_base_v3& src ) ; - -#if __TBB_CPP11_RVALUE_REF_PRESENT - //! swap queues - void internal_swap( concurrent_queue_base_v3& src ) { - std::swap( my_capacity, src.my_capacity ); - std::swap( items_per_page, src.items_per_page ); - std::swap( item_size, src.item_size ); - std::swap( my_rep, src.my_rep ); - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - //! Enqueues item at tail of queue using specified operation (copy or move) - void internal_insert_item( const void* src, copy_specifics op_type ); - - //! Attempts to enqueue at tail of queue using specified operation (copy or move) - bool internal_insert_if_not_full( const void* src, copy_specifics op_type ); - - //! Assigns one queue to another using specified operation (copy or move) - void internal_assign( const concurrent_queue_base_v3& src, copy_specifics op_type ); -private: - virtual void copy_page_item( page& dst, size_t dindex, const page& src, size_t sindex ) = 0; -}; - -//! For internal use only. -/** Backward compatible modification of concurrent_queue_base_v3 - @ingroup containers */ -class concurrent_queue_base_v8: public concurrent_queue_base_v3 { -protected: - concurrent_queue_base_v8( size_t item_sz ) : concurrent_queue_base_v3( item_sz ) {} - - //! move items - void __TBB_EXPORTED_METHOD move_content( concurrent_queue_base_v8& src ) ; - - //! Attempt to enqueue item onto queue using move operation - bool __TBB_EXPORTED_METHOD internal_push_move_if_not_full( const void* src ); - - //! Enqueue item at tail of queue using move operation - void __TBB_EXPORTED_METHOD internal_push_move( const void* src ); -private: - friend struct micro_queue; - virtual void move_page_item( page& dst, size_t dindex, const page& src, size_t sindex ) = 0; - virtual void move_item( page& dst, size_t index, const void* src ) = 0; -}; - -//! Type-independent portion of concurrent_queue_iterator. -/** @ingroup containers */ -class concurrent_queue_iterator_base_v3 { - //! concurrent_queue over which we are iterating. - /** NULL if one past last element in queue. */ - concurrent_queue_iterator_rep* my_rep; - - template - friend bool operator==( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ); - - template - friend bool operator!=( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ); - - void initialize( const concurrent_queue_base_v3& queue, size_t offset_of_data ); -protected: - //! Pointer to current item - void* my_item; - - //! Default constructor - concurrent_queue_iterator_base_v3() : my_rep(NULL), my_item(NULL) {} - - //! Copy constructor - concurrent_queue_iterator_base_v3( const concurrent_queue_iterator_base_v3& i ) : my_rep(NULL), my_item(NULL) { - assign(i); - } - - concurrent_queue_iterator_base_v3& operator=( const concurrent_queue_iterator_base_v3& i ) { - assign(i); - return *this; - } - - //! Obsolete entry point for constructing iterator pointing to head of queue. - /** Does not work correctly for SSE types. */ - __TBB_EXPORTED_METHOD concurrent_queue_iterator_base_v3( const concurrent_queue_base_v3& queue ); - - //! Construct iterator pointing to head of queue. - __TBB_EXPORTED_METHOD concurrent_queue_iterator_base_v3( const concurrent_queue_base_v3& queue, size_t offset_of_data ); - - //! Assignment - void __TBB_EXPORTED_METHOD assign( const concurrent_queue_iterator_base_v3& i ); - - //! Advance iterator one step towards tail of queue. - void __TBB_EXPORTED_METHOD advance(); - - //! Destructor - __TBB_EXPORTED_METHOD ~concurrent_queue_iterator_base_v3(); -}; - -typedef concurrent_queue_iterator_base_v3 concurrent_queue_iterator_base; - -//! Meets requirements of a forward iterator for STL. -/** Value is either the T or const T type of the container. - @ingroup containers */ -template -class concurrent_queue_iterator: public concurrent_queue_iterator_base, - public std::iterator { - -#if !__TBB_TEMPLATE_FRIENDS_BROKEN - template - friend class ::tbb::concurrent_bounded_queue; -#else -public: -#endif - - //! Construct iterator pointing to head of queue. - explicit concurrent_queue_iterator( const concurrent_queue_base_v3& queue ) : - concurrent_queue_iterator_base_v3(queue,__TBB_offsetof(concurrent_queue_base_v3::padded_page,last)) - { - } - -public: - concurrent_queue_iterator() {} - - /** If Value==Container::value_type, then this routine is the copy constructor. - If Value==const Container::value_type, then this routine is a conversion constructor. */ - concurrent_queue_iterator( const concurrent_queue_iterator& other ) : - concurrent_queue_iterator_base_v3(other) - {} - - //! Iterator assignment - concurrent_queue_iterator& operator=( const concurrent_queue_iterator& other ) { - concurrent_queue_iterator_base_v3::operator=(other); - return *this; - } - - //! Reference to current item - Value& operator*() const { - return *static_cast(my_item); - } - - Value* operator->() const {return &operator*();} - - //! Advance to next item in queue - concurrent_queue_iterator& operator++() { - advance(); - return *this; - } - - //! Post increment - Value* operator++(int) { - Value* result = &operator*(); - operator++(); - return result; - } -}; // concurrent_queue_iterator - - -template -bool operator==( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ) { - return i.my_item==j.my_item; -} - -template -bool operator!=( const concurrent_queue_iterator& i, const concurrent_queue_iterator& j ) { - return i.my_item!=j.my_item; -} - -} // namespace internal; - -//! @endcond - -} // namespace tbb - -#endif /* __TBB__concurrent_queue_impl_H */ diff --git a/src/3rd party/tbb/internal/_concurrent_skip_list_impl.h b/src/3rd party/tbb/internal/_concurrent_skip_list_impl.h deleted file mode 100644 index ed9abc5f28..0000000000 --- a/src/3rd party/tbb/internal/_concurrent_skip_list_impl.h +++ /dev/null @@ -1,1085 +0,0 @@ -/* - Copyright (c) 2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_concurrent_skip_list_H -#define __TBB_concurrent_skip_list_H - -#if !defined(__TBB_concurrent_map_H) && !defined(__TBB_concurrent_set_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include "../tbb_config.h" -#include "../tbb_stddef.h" -#include "../tbb_allocator.h" -#include "../spin_mutex.h" -#include "../tbb_exception.h" -#include "../enumerable_thread_specific.h" -#include "_allocator_traits.h" -#include "_template_helpers.h" -#include "_node_handle_impl.h" -#include // Need std::pair -#include -#include -#include // Need std::allocator_traits -#include -#include -#include -#include -#include -#include -#include -#include - -#if _MSC_VER -#pragma warning(disable: 4189) // warning 4189 -- local variable is initialized but not referenced -#pragma warning(disable: 4127) // warning 4127 -- while (true) has a constant expression in it -#endif - -namespace tbb { -namespace interface10 { -namespace internal { - -template -class skip_list_node { - -public: - using value_type = Value; - using size_type = std::size_t; - using reference = value_type & ; - using const_reference = const value_type & ; - using pointer = value_type * ; - using const_pointer = const value_type *; - using node_pointer = skip_list_node * ; - using atomic_node_pointer = std::atomic; - - using mutex_type = Mutex; - using lock_type = std::unique_lock; - - skip_list_node(size_type levels) : my_height(levels), my_fullyLinked(false) { - for (size_type lev = 0; lev < my_height; ++lev) - new(&my_next(lev)) atomic_node_pointer(nullptr); - __TBB_ASSERT(height() == levels, "Wrong node height"); - } - - ~skip_list_node() { - for(size_type lev = 0; lev < my_height; ++lev) - my_next(lev).~atomic(); - } - - skip_list_node(const skip_list_node&) = delete; - - skip_list_node(skip_list_node&&) = delete; - - skip_list_node& operator=(const skip_list_node&) = delete; - - pointer storage() { - return reinterpret_cast(&my_val); - } - - reference value() { - return *storage(); - } - - node_pointer next(size_type level) const { - __TBB_ASSERT(level < height(), "Cannot get next on the level greater than height"); - return my_next(level).load(std::memory_order_acquire); - } - - void set_next(size_type level, node_pointer next) { - __TBB_ASSERT(level < height(), "Cannot set next on the level greater than height"); - - my_next(level).store(next, std::memory_order_release); - } - - /** @return number of layers */ - size_type height() const { - return my_height; - } - - bool fully_linked() const { - return my_fullyLinked.load(std::memory_order_acquire); - } - - void mark_linked() { - my_fullyLinked.store(true, std::memory_order_release); - } - - lock_type acquire() { - return lock_type(my_mutex); - } - -private: - using aligned_storage_type = typename std::aligned_storage::type; - - atomic_node_pointer& my_next(size_type level) { - atomic_node_pointer* arr = reinterpret_cast(this + 1); - return arr[level]; - } - - const atomic_node_pointer& my_next(size_type level) const { - const atomic_node_pointer* arr = reinterpret_cast(this + 1); - return arr[level]; - } - - mutex_type my_mutex; - aligned_storage_type my_val; - size_type my_height; - std::atomic_bool my_fullyLinked; -}; - -template -class skip_list_iterator { - using node_type = NodeType; - using node_ptr = node_type*; -public: - using iterator_category = std::forward_iterator_tag; - using value_type = typename node_type::value_type; - using difference_type = std::ptrdiff_t; - using pointer = typename std::conditional::type; - using reference = typename std::conditional::type; - - skip_list_iterator() : my_node_ptr(nullptr) {} - - // TODO: the code above does not compile in VS2015 (seems like a bug) - consider enabling it for all other platforms - // template ::type> - // skip_list_iterator(const skip_list_iterator& other) : my_node_ptr(other.my_node_ptr) {} - - // skip_list_iterator(const skip_list_iterator& other) : my_node_ptr(other.my_node_ptr) {} - - skip_list_iterator(const skip_list_iterator& other) : my_node_ptr(other.my_node_ptr) {} - - skip_list_iterator& operator=(const skip_list_iterator& other) { - my_node_ptr = other.my_node_ptr; - return *this; - } - - template ::type> - skip_list_iterator(const skip_list_iterator& other) : my_node_ptr(other.my_node_ptr) {} - - reference operator*() const { return *(my_node_ptr->storage()); } - pointer operator->() const { return &**this; } - - skip_list_iterator& operator++() { - __TBB_ASSERT(my_node_ptr != nullptr, NULL); - my_node_ptr = my_node_ptr->next(0); - return *this; - } - - skip_list_iterator operator++(int) { - skip_list_iterator tmp = *this; - ++*this; - return tmp; - } - -private: - skip_list_iterator(node_type* n) : my_node_ptr(n) {} - - node_ptr my_node_ptr; - - template - friend class concurrent_skip_list; - - friend class skip_list_iterator; - - friend class const_range; - friend class range; - - template - friend bool operator==(const skip_list_iterator&, const skip_list_iterator&); - - template - friend bool operator!=(const skip_list_iterator&, const skip_list_iterator&); -}; - -template -bool operator==(const skip_list_iterator& lhs, const skip_list_iterator& rhs) { - return lhs.my_node_ptr == rhs.my_node_ptr; -} - -template -bool operator!=(const skip_list_iterator& lhs, const skip_list_iterator& rhs) { - return lhs.my_node_ptr != rhs.my_node_ptr; -} - -template -class concurrent_skip_list { -protected: - using traits_type = Traits; - using allocator_type = typename traits_type::allocator_type; - using allocator_traits_type = std::allocator_traits; - using key_compare = typename traits_type::compare_type; - using value_compare = typename traits_type::value_compare; - using key_type = typename traits_type::key_type; - using value_type = typename traits_type::value_type; - using node_type = typename traits_type::node_type; - using list_node_type = skip_list_node; - - using iterator = skip_list_iterator; - using const_iterator = skip_list_iterator; - using reverse_iterator = std::reverse_iterator; - using const_reverse_iterator = std::reverse_iterator; - - using reference = value_type&; - using const_reference = const value_type&; - using pointer = typename allocator_traits_type::pointer; - using const_pointer = typename allocator_traits_type::const_pointer; - using size_type = std::size_t; - using difference_type = std::ptrdiff_t; - - using random_level_generator_type = typename traits_type::random_level_generator_type; - using node_allocator_type = typename std::allocator_traits::template rebind_alloc; - using node_allocator_traits = typename std::allocator_traits::template rebind_traits; - using node_ptr = list_node_type*; - - static constexpr size_type MAX_LEVEL = traits_type::MAX_LEVEL; - - using array_type = std::array; - using lock_array = std::array; - -public: - static bool const allow_multimapping = traits_type::allow_multimapping; - - /** - * Default constructor. Construct empty skip list. - */ - concurrent_skip_list() : my_size(0) { - create_dummy_head(); - } - - explicit concurrent_skip_list(const key_compare& comp, const allocator_type& alloc = allocator_type()) - : my_node_allocator(alloc), my_compare(comp), my_size(0) - { - create_dummy_head(); - } - - template - concurrent_skip_list(InputIt first, InputIt last, const key_compare& comp = key_compare(), - const allocator_type& alloc = allocator_type()) - : my_node_allocator(alloc), my_compare(comp), my_size(0) - { - create_dummy_head(); - internal_copy(first, last); - } - - /** Copy constructor */ - concurrent_skip_list(const concurrent_skip_list& other) - : my_node_allocator(node_allocator_traits::select_on_container_copy_construction(other.get_allocator())), - my_compare(other.my_compare), my_rnd_generator(other.my_rnd_generator), my_size(0) - { - create_dummy_head(); - internal_copy(other); - __TBB_ASSERT(my_size == other.my_size, "Wrong size of copy-constructed container"); - } - - concurrent_skip_list(const concurrent_skip_list& other, const allocator_type& alloc) - : my_node_allocator(alloc), my_compare(other.my_compare), - my_rnd_generator(other.my_rnd_generator), my_size(0) - { - create_dummy_head(); - internal_copy(other); - __TBB_ASSERT(my_size == other.my_size, "Wrong size of copy-constructed container"); - } - - concurrent_skip_list(concurrent_skip_list&& other) - : my_node_allocator(std::move(other.my_node_allocator)), my_compare(other.my_compare), - my_rnd_generator(other.my_rnd_generator) - { - internal_move(std::move(other)); - } - - concurrent_skip_list(concurrent_skip_list&& other, const allocator_type& alloc) - : my_node_allocator(alloc), my_compare(other.my_compare), - my_rnd_generator(other.my_rnd_generator) - { - if (alloc == other.get_allocator()) { - internal_move(std::move(other)); - } else { - my_size = 0; - create_dummy_head(); - internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); - } - } - - ~concurrent_skip_list() { - clear(); - delete_dummy_head(); - } - - concurrent_skip_list& operator=(const concurrent_skip_list& other) { - if (this != &other) { - using pocca_type = typename node_allocator_traits::propagate_on_container_copy_assignment; - clear(); - tbb::internal::allocator_copy_assignment(my_node_allocator, other.my_node_allocator, pocca_type()); - my_compare = other.my_compare; - my_rnd_generator = other.my_rnd_generator; - internal_copy(other); - } - return *this; - } - - concurrent_skip_list& operator=(concurrent_skip_list&& other) { - if (this != &other) { - using pocma_type = typename node_allocator_traits::propagate_on_container_move_assignment; - clear(); - my_compare = other.my_compare; - my_rnd_generator = other.my_rnd_generator; - internal_move_assign(std::move(other), pocma_type()); - } - return *this; - } - - concurrent_skip_list& operator=(std::initializer_list il) - { - clear(); - insert(il.begin(),il.end()); - return *this; - } - - std::pair insert(const value_type& value) { - return internal_insert(value); - } - - std::pair insert(value_type&& value) { - return internal_insert(std::move(value)); - } - - iterator insert(const_iterator, const_reference value) { - // Ignore hint - return insert(value).first; - } - - iterator insert(const_iterator, value_type&& value) { - // Ignore hint - return insert(std::move(value)).first; - } - - template - void insert(InputIterator first, InputIterator last) { - for (InputIterator it = first; it != last; ++it) - insert(*it); - } - - void insert(std::initializer_list init) { - insert(init.begin(), init.end()); - } - - std::pair insert(node_type&& nh) { - if(!nh.empty()) { - std::pair insert_result = internal_insert_node(nh.my_node); - if(insert_result.second) { - nh.deactivate(); - } - return insert_result; - } - return std::pair(end(), false); - } - - iterator insert(const_iterator, node_type&& nh) { - // Ignore hint - return insert(std::move(nh)).first; - } - - template - std::pair emplace(Args&&... args) { - return internal_insert(std::forward(args)...); - } - - template - iterator emplace_hint(const_iterator, Args&&... args) { - // Ignore hint - return emplace(std::forward(args)...).first; - } - - iterator unsafe_erase(iterator pos) { - std::pair extract_result = internal_extract(pos); - if(extract_result.first) { // node was extracted - delete_node(extract_result.first); - return iterator(extract_result.second); - } - return end(); - } - - iterator unsafe_erase(const_iterator pos) { - return unsafe_erase(get_iterator(pos)); - } - - template , - typename = typename std::enable_if::value && - !std::is_convertible::value>::type> - size_type unsafe_erase(const K& key) { - std::pair range = equal_range(key); - size_type sz = std::distance(range.first, range.second); - unsafe_erase(range.first, range.second); - return sz; - } - - iterator unsafe_erase(const_iterator first, const_iterator last) { - while(first != last) { - first = unsafe_erase(get_iterator(first)); - } - return get_iterator(first); - } - - size_type unsafe_erase(const key_type& key) { - std::pair range = equal_range(key); - size_type sz = std::distance(range.first, range.second); - unsafe_erase(range.first, range.second); - return sz; - } - - node_type unsafe_extract(const_iterator pos) { - std::pair extract_result = internal_extract(pos); - return extract_result.first ? node_type(extract_result.first) : node_type(); - } - - node_type unsafe_extract(const key_type& key) { - return unsafe_extract(find(key)); - } - - iterator lower_bound(const key_type& key) { - return internal_get_bound(key, my_compare); - } - - const_iterator lower_bound(const key_type& key) const { - return internal_get_bound(key, my_compare); - } - - template > - iterator lower_bound(const K& key) { - return internal_get_bound(key, my_compare); - } - - template > - const_iterator lower_bound(const K& key) const { - return internal_get_bound(key, my_compare); - } - - iterator upper_bound(const key_type& key) { - return internal_get_bound(key, not_greater_compare(my_compare)); - } - - const_iterator upper_bound(const key_type& key) const { - return internal_get_bound(key, not_greater_compare(my_compare)); - } - - template > - iterator upper_bound(const K& key) { - return internal_get_bound(key, not_greater_compare(my_compare)); - } - - template > - const_iterator upper_bound(const K& key) const { - return internal_get_bound(key, not_greater_compare(my_compare)); - } - - iterator find(const key_type& key) { - return internal_find(key); - } - - const_iterator find(const key_type& key) const { - return internal_find(key); - } - - template > - iterator find(const K& key) { - return internal_find(key); - } - - template > - const_iterator find(const K& key) const { - return internal_find(key); - } - - size_type count( const key_type& key ) const { - return internal_count(key); - } - - template > - size_type count(const K& key) const { - return internal_count(key); - } - - bool contains(const key_type& key) const { - return find(key) != end(); - } - - template > - bool contains(const K& key) const { - return find(key) != end(); - } - - void clear() noexcept { - __TBB_ASSERT(dummy_head->height() > 0, NULL); - - node_ptr current = dummy_head->next(0); - while (current) { - __TBB_ASSERT(current->height() > 0, NULL); - node_ptr next = current->next(0); - delete_node(current); - current = next; - } - - my_size = 0; - for (size_type i = 0; i < dummy_head->height(); ++i) { - dummy_head->set_next(i, nullptr); - } - } - - iterator begin() { - return iterator(dummy_head->next(0)); - } - - const_iterator begin() const { - return const_iterator(dummy_head->next(0)); - } - - const_iterator cbegin() const { - return const_iterator(dummy_head->next(0)); - } - - iterator end() { - return iterator(nullptr); - } - - const_iterator end() const { - return const_iterator(nullptr); - } - - const_iterator cend() const { - return const_iterator(nullptr); - } - - size_type size() const { - return my_size.load(std::memory_order_relaxed); - } - - size_type max_size() const { - return my_node_allocator.max_size(); - } - - bool empty() const { - return 0 == size(); - } - - allocator_type get_allocator() const { - return my_node_allocator; - } - - void swap(concurrent_skip_list& other) { - using std::swap; - using pocs_type = typename node_allocator_traits::propagate_on_container_swap; - tbb::internal::allocator_swap(my_node_allocator, other.my_node_allocator, pocs_type()); - swap(my_compare, other.my_compare); - swap(my_rnd_generator, other.my_rnd_generator); - swap(dummy_head, other.dummy_head); - - size_type tmp = my_size; - my_size.store(other.my_size); - other.my_size.store(tmp); - } - - std::pair equal_range(const key_type& key) { - return std::pair(lower_bound(key), upper_bound(key)); - } - - std::pair equal_range(const key_type& key) const { - return std::pair(lower_bound(key), upper_bound(key)); - } - - template > - std::pair equal_range(const K& key) { - return std::pair(lower_bound(key), upper_bound(key)); - } - - template > - std::pair equal_range(const K& key) const { - return std::pair(lower_bound(key), upper_bound(key)); - } - - key_compare key_comp() const { return my_compare; } - - value_compare value_comp() const { return traits_type::value_comp(my_compare); } - - class const_range_type : tbb::internal::no_assign { - public: - using size_type = typename concurrent_skip_list::size_type; - using value_type = typename concurrent_skip_list::value_type; - using iterator = typename concurrent_skip_list::const_iterator; - private: - const_iterator my_end; - const_iterator my_begin; - size_type my_level; - - public: - - bool empty() const { - return my_begin.my_node_ptr->next(0) == my_end.my_node_ptr; - } - - bool is_divisible() const { - return my_level != 0 ? my_begin.my_node_ptr->next(my_level - 1) != my_end.my_node_ptr : false; - } - - size_type size() const { return std::distance(my_begin, my_end);} - - const_range_type( const_range_type& r, split) - : my_end(r.my_end) { - my_begin = iterator(r.my_begin.my_node_ptr->next(r.my_level - 1)); - my_level = my_begin.my_node_ptr->height(); - r.my_end = my_begin; - } - - const_range_type( const concurrent_skip_list& l) - : my_end(l.end()), my_begin(l.begin()), my_level(my_begin.my_node_ptr->height() ) {} - - iterator begin() const { return my_begin; } - iterator end() const { return my_end; } - size_t grainsize() const { return 1; } - - }; // class const_range_type - - class range_type : public const_range_type { - public: - using iterator = typename concurrent_skip_list::iterator; - - range_type(range_type& r, split) : const_range_type(r, split()) {} - range_type(const concurrent_skip_list& l) : const_range_type(l) {} - - iterator begin() const { - node_ptr node = const_range_type::begin().my_node_ptr; - return iterator(node); - } - - iterator end() const { - node_ptr node = const_range_type::end().my_node_ptr; - return iterator(node); } - }; // class range_type - - range_type range() { return range_type(*this); } - const_range_type range() const { return const_range_type(*this); } - -private: - void internal_move(concurrent_skip_list&& other) { - dummy_head = other.dummy_head; - other.dummy_head = nullptr; - other.create_dummy_head(); - - my_size = other.my_size.load(); - other.my_size = 0; - } - - static const key_type& get_key(node_ptr n) { - __TBB_ASSERT(n, NULL); - return traits_type::get_key(n->value()); - } - - template - iterator internal_find(const K& key) { - iterator it = lower_bound(key); - return (it == end() || my_compare(key, traits_type::get_key(*it))) ? end() : it; - } - - template - const_iterator internal_find(const K& key) const { - const_iterator it = lower_bound(key); - return (it == end() || my_compare(key, traits_type::get_key(*it))) ? end() : it; - } - - template - size_type internal_count( const K& key ) const { - if (allow_multimapping) { - std::pair range = equal_range(key); - return std::distance(range.first, range.second); - } - return (find(key) == end()) ? size_type(0) : size_type(1); - } - - /** - * Finds position on the @param level using @param cmp - * @param level - on which level search prev node - * @param prev - pointer to the start node to search - * @param key - key to search - * @param cmp - callable object to compare two objects - * (my_compare member is default comparator) - * @returns pointer to the node which is not satisfy the comparison with @param key - */ - template - pointer_type internal_find_position( size_type level, pointer_type& prev, const K& key, - const comparator& cmp) const { - __TBB_ASSERT(level < prev->height(), "Wrong level to find position"); - pointer_type curr = prev->next(level); - - while (curr && cmp(get_key(curr), key)) { - prev = curr; - __TBB_ASSERT(level < prev->height(), NULL); - curr = prev->next(level); - } - - return curr; - } - - template - void fill_prev_next_arrays(array_type& prev_nodes, array_type& next_nodes, node_ptr prev, const key_type& key, - const comparator& cmp) { - prev_nodes.fill(dummy_head); - next_nodes.fill(nullptr); - - for (size_type h = prev->height(); h > 0; --h) { - node_ptr next = internal_find_position(h - 1, prev, key, cmp); - prev_nodes[h - 1] = prev; - next_nodes[h - 1] = next; - } - } - - template - void fill_prev_next_by_ptr(array_type& prev_nodes, array_type& next_nodes, const_iterator it, const key_type& key, - const comparator& cmp) { - node_ptr prev = dummy_head; - node_ptr erase_node = it.my_node_ptr; - size_type node_height = erase_node->height(); - - for (size_type h = prev->height(); h >= node_height; --h) { - internal_find_position(h - 1, prev, key, cmp); - } - - for (size_type h = node_height; h > 0; --h) { - node_ptr curr = prev->next(h - 1); - while (const_iterator(curr) != it) { - prev = curr; - curr = prev->next(h - 1); - } - prev_nodes[h - 1] = prev; - } - - std::fill(next_nodes.begin(), next_nodes.begin() + node_height, erase_node); - } - - template - std::pair internal_insert(Args&&... args) { - node_ptr new_node = create_node(std::forward(args)...); - std::pair insert_result = internal_insert_node(new_node); - if(!insert_result.second) { - delete_node(new_node); - } - return insert_result; - } - - std::pair internal_insert_node(node_ptr new_node) { - array_type prev_nodes; - array_type next_nodes; - __TBB_ASSERT(dummy_head->height() >= new_node->height(), "Wrong height for new node"); - - do { - if (allow_multimapping) { - fill_prev_next_arrays(prev_nodes, next_nodes, dummy_head, get_key(new_node), - not_greater_compare(my_compare)); - } else { - fill_prev_next_arrays(prev_nodes, next_nodes, dummy_head, get_key(new_node), my_compare); - } - - node_ptr next = next_nodes[0]; - if (next && !allow_multimapping && !my_compare(get_key(new_node), get_key(next))) { - // TODO: do we really need to wait? - while (!next->fully_linked()) { - // TODO: atomic backoff - } - - return std::pair(iterator(next), false); - } - __TBB_ASSERT(allow_multimapping || !next || my_compare(get_key(new_node), get_key(next)), - "Wrong elements order"); - - } while (!try_insert_node(new_node, prev_nodes, next_nodes)); - - __TBB_ASSERT(new_node, NULL); - return std::pair(iterator(new_node), true); - } - - bool try_insert_node(node_ptr new_node, array_type& prev_nodes, array_type& next_nodes) { - __TBB_ASSERT(dummy_head->height() >= new_node->height(), NULL); - - lock_array locks; - - if (!try_lock_nodes(new_node->height(), prev_nodes, next_nodes, locks)) { - return false; - } - - __TBB_ASSERT(allow_multimapping || - ((prev_nodes[0] == dummy_head || - my_compare(get_key(prev_nodes[0]), get_key(new_node))) && - (next_nodes[0] == nullptr || my_compare(get_key(new_node), get_key(next_nodes[0])))), - "Wrong elements order"); - - for (size_type level = 0; level < new_node->height(); ++level) { - __TBB_ASSERT(prev_nodes[level]->height() > level, NULL); - __TBB_ASSERT(prev_nodes[level]->next(level) == next_nodes[level], NULL); - new_node->set_next(level, next_nodes[level]); - prev_nodes[level]->set_next(level, new_node); - } - new_node->mark_linked(); - - ++my_size; - - return true; - } - - bool try_lock_nodes(size_type height, array_type& prevs, array_type& next_nodes, lock_array& locks) { - for (size_type l = 0; l < height; ++l) { - if (l == 0 || prevs[l] != prevs[l - 1]) - locks[l] = prevs[l]->acquire(); - - node_ptr next = prevs[l]->next(l); - if ( next != next_nodes[l]) return false; - } - - return true; - } - - template - const_iterator internal_get_bound(const K& key, const comparator& cmp) const { - node_ptr prev = dummy_head; - __TBB_ASSERT(dummy_head->height() > 0, NULL); - node_ptr next = nullptr; - - for (size_type h = prev->height(); h > 0; --h) { - next = internal_find_position(h - 1, prev, key, cmp); - } - - return const_iterator(next); - } - - template - iterator internal_get_bound(const K& key, const comparator& cmp){ - node_ptr prev = dummy_head; - __TBB_ASSERT(dummy_head->height() > 0, NULL); - node_ptr next = nullptr; - - for (size_type h = prev->height(); h > 0; --h) { - next = internal_find_position(h - 1, prev, key, cmp); - } - - return iterator(next); - } - - // Returns node_ptr to the extracted node and node_ptr to the next node after the extracted - std::pair internal_extract(const_iterator it) { - if ( it != end() ) { - key_type key = traits_type::get_key(*it); - __TBB_ASSERT(dummy_head->height() > 0, NULL); - - array_type prev_nodes; - array_type next_nodes; - - fill_prev_next_by_ptr(prev_nodes, next_nodes, it, key, my_compare); - - node_ptr erase_node = next_nodes[0]; - __TBB_ASSERT(erase_node != nullptr, NULL); - node_ptr next_node = erase_node->next(0); - - if (!my_compare(key, get_key(erase_node))) { - for(size_type level = 0; level < erase_node->height(); ++level) { - __TBB_ASSERT(prev_nodes[level]->height() > level, NULL); - __TBB_ASSERT(next_nodes[level] == erase_node, NULL); - prev_nodes[level]->set_next(level, erase_node->next(level)); - } - --my_size; - return std::pair(erase_node, next_node); - } - } - return std::pair(nullptr, nullptr); - } - -protected: - template - void internal_merge(SourceType&& source) { - using source_type = typename std::decay::type; - using source_iterator = typename source_type::iterator; - __TBB_STATIC_ASSERT((std::is_same::value), "Incompatible containers cannot be merged"); - - for(source_iterator it = source.begin(); it != source.end();) { - source_iterator where = it++; - if (allow_multimapping || !contains(traits_type::get_key(*where))) { - std::pair extract_result = source.internal_extract(where); - - //If the insertion fails - return the node into source - node_type handle(extract_result.first); - __TBB_ASSERT(!handle.empty(), "Extracted handle in merge is empty"); - - if (!insert(std::move(handle)).second) { - source.insert(std::move(handle)); - } - handle.deactivate(); - } - } - } - -private: - void internal_copy(const concurrent_skip_list& other) { - internal_copy(other.begin(), other.end()); - } - - template - void internal_copy(Iterator first, Iterator last) { - clear(); - try { - for (auto it = first; it != last; ++it) - insert(*it); - } - catch (...) { - clear(); - delete_dummy_head(); - throw; - } - } - - /** Generate random level */ - size_type random_level() { - return my_rnd_generator(); - } - - static size_type calc_node_size(size_type height) { - return sizeof(list_node_type) + height*sizeof(typename list_node_type::atomic_node_pointer); - } - - /** Creates new node */ - template - node_ptr create_node(Args&&... args) { - size_type levels = random_level(); - - size_type sz = calc_node_size(levels); - - node_ptr node = reinterpret_cast(node_allocator_traits::allocate(my_node_allocator, sz)); - - try { - node_allocator_traits::construct(my_node_allocator, node, levels); - - } - catch(...) { - deallocate_node(node, sz); - throw; - } - - try { - node_allocator_traits::construct(my_node_allocator, node->storage(), std::forward(args)...); - } - catch (...) { - node_allocator_traits::destroy(my_node_allocator, node); - deallocate_node(node, sz); - throw; - } - - return node; - } - - void create_dummy_head() { - size_type sz = calc_node_size(MAX_LEVEL); - - dummy_head = reinterpret_cast(node_allocator_traits::allocate(my_node_allocator, sz)); - // TODO: investigate linkage fail in debug without this workaround - auto max_level = MAX_LEVEL; - - try { - node_allocator_traits::construct(my_node_allocator, dummy_head, max_level); - } - catch(...) { - deallocate_node(dummy_head, sz); - throw; - } - } - - template - void delete_node(node_ptr node) { - size_type sz = calc_node_size(node->height()); - // Destroy value - if (!is_dummy) node_allocator_traits::destroy(my_node_allocator, node->storage()); - // Destroy node - node_allocator_traits::destroy(my_node_allocator, node); - // Deallocate memory - deallocate_node(node, sz); - } - - void deallocate_node(node_ptr node, size_type sz) { - node_allocator_traits::deallocate(my_node_allocator, reinterpret_cast(node), sz); - } - - void delete_dummy_head() { - delete_node(dummy_head); - } - - static iterator get_iterator(const_iterator it) { - return iterator(it.my_node_ptr); - } - - void internal_move_assign(concurrent_skip_list&& other, /*POCMA=*/std::true_type) { - delete_dummy_head(); - tbb::internal::allocator_move_assignment(my_node_allocator, other.my_node_allocator, std::true_type()); - internal_move(std::move(other)); - } - - void internal_move_assign(concurrent_skip_list&& other, /*POCMA=*/std::false_type) { - if (my_node_allocator == other.my_node_allocator) { - delete_dummy_head(); - internal_move(std::move(other)); - } else { - internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); - } - } - - struct not_greater_compare { - const key_compare& my_less_compare; - - not_greater_compare(const key_compare& less_compare) : my_less_compare(less_compare) {} - - template - bool operator()(const K1& first, const K2& second) const { - return !my_less_compare(second, first); - } - }; - - node_allocator_type my_node_allocator; - key_compare my_compare; - random_level_generator_type my_rnd_generator; - node_ptr dummy_head; - - template - friend class concurrent_skip_list; - - std::atomic my_size; -}; // class concurrent_skip_list - -template -class concurrent_geometric_level_generator { -public: - static constexpr size_t max_level = MAX_LEVEL; - - concurrent_geometric_level_generator() : engines(time(NULL)) {} - - size_t operator()() { - return (distribution(engines.local()) % MAX_LEVEL) + 1; - } - -private: - tbb::enumerable_thread_specific engines; - std::geometric_distribution distribution; -}; - -} // namespace internal -} // namespace interface10 -} // namespace tbb - -#endif // __TBB_concurrent_skip_list_H diff --git a/src/3rd party/tbb/internal/_concurrent_unordered_impl.h b/src/3rd party/tbb/internal/_concurrent_unordered_impl.h deleted file mode 100644 index 21b35da9b1..0000000000 --- a/src/3rd party/tbb/internal/_concurrent_unordered_impl.h +++ /dev/null @@ -1,1684 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* Container implementations in this header are based on PPL implementations - provided by Microsoft. */ - -#ifndef __TBB__concurrent_unordered_impl_H -#define __TBB__concurrent_unordered_impl_H -#if !defined(__TBB_concurrent_unordered_map_H) && !defined(__TBB_concurrent_unordered_set_H) && !defined(__TBB_concurrent_hash_map_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include "../tbb_stddef.h" - -#include -#include // Need std::pair -#include // Need std::equal_to (in ../concurrent_unordered_*.h) -#include // For tbb_hasher -#include // Need std::memset -#include __TBB_STD_SWAP_HEADER - -#include "../atomic.h" -#include "../tbb_exception.h" -#include "../tbb_allocator.h" - -#if __TBB_INITIALIZER_LISTS_PRESENT - #include -#endif - -#if __TBB_CPP11_RVALUE_REF_PRESENT && !__TBB_IMPLICIT_COPY_DELETION_BROKEN - #define __TBB_UNORDERED_NODE_HANDLE_PRESENT 1 -#endif - -#include "_allocator_traits.h" -#include "_tbb_hash_compare_impl.h" -#include "_template_helpers.h" - -#if __TBB_UNORDERED_NODE_HANDLE_PRESENT -#include "_node_handle_impl.h" -#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT - -namespace tbb { -namespace interface5 { -//! @cond INTERNAL -namespace internal { - -template -class split_ordered_list; -template -class concurrent_unordered_base; - -// Forward list iterators (without skipping dummy elements) -template -class flist_iterator : public std::iterator -{ - template - friend class split_ordered_list; - template - friend class concurrent_unordered_base; - template - friend class flist_iterator; - - typedef typename Solist::nodeptr_t nodeptr_t; -public: - typedef typename Solist::value_type value_type; - typedef typename Solist::difference_type difference_type; - typedef typename Solist::pointer pointer; - typedef typename Solist::reference reference; - - flist_iterator() : my_node_ptr(0) {} - flist_iterator( const flist_iterator &other ) - : my_node_ptr(other.my_node_ptr) {} - - flist_iterator& operator=( const flist_iterator &other ) { - my_node_ptr = other.my_node_ptr; - return *this; - } - - reference operator*() const { return my_node_ptr->my_element; } - pointer operator->() const { return &**this; } - - flist_iterator& operator++() { - my_node_ptr = my_node_ptr->my_next; - return *this; - } - - flist_iterator operator++(int) { - flist_iterator tmp = *this; - ++*this; - return tmp; - } - -protected: - flist_iterator(nodeptr_t pnode) : my_node_ptr(pnode) {} - nodeptr_t get_node_ptr() const { return my_node_ptr; } - - nodeptr_t my_node_ptr; - - template - friend bool operator==( const flist_iterator &i, const flist_iterator &j ); - template - friend bool operator!=( const flist_iterator& i, const flist_iterator& j ); -}; - -template -bool operator==( const flist_iterator &i, const flist_iterator &j ) { - return i.my_node_ptr == j.my_node_ptr; -} -template -bool operator!=( const flist_iterator& i, const flist_iterator& j ) { - return i.my_node_ptr != j.my_node_ptr; -} - -// Split-order list iterators, needed to skip dummy elements -template -class solist_iterator : public flist_iterator -{ - typedef flist_iterator base_type; - typedef typename Solist::nodeptr_t nodeptr_t; - using base_type::get_node_ptr; - template - friend class split_ordered_list; - template - friend class solist_iterator; - template - friend class concurrent_unordered_base; - template - friend bool operator==( const solist_iterator &i, const solist_iterator &j ); - template - friend bool operator!=( const solist_iterator& i, const solist_iterator& j ); - - const Solist *my_list_ptr; - solist_iterator(nodeptr_t pnode, const Solist *plist) : base_type(pnode), my_list_ptr(plist) {} - -public: - typedef typename Solist::value_type value_type; - typedef typename Solist::difference_type difference_type; - typedef typename Solist::pointer pointer; - typedef typename Solist::reference reference; - - solist_iterator() {} - solist_iterator( const solist_iterator &other ) - : base_type(other), my_list_ptr(other.my_list_ptr) {} - - solist_iterator& operator=( const solist_iterator &other ) { - base_type::my_node_ptr = other.get_node_ptr(); - my_list_ptr = other.my_list_ptr; - return *this; - } - - reference operator*() const { - return this->base_type::operator*(); - } - - pointer operator->() const { - return (&**this); - } - - solist_iterator& operator++() { - do ++(*(base_type *)this); - while (get_node_ptr() != NULL && get_node_ptr()->is_dummy()); - - return (*this); - } - - solist_iterator operator++(int) { - solist_iterator tmp = *this; - do ++*this; - while (get_node_ptr() != NULL && get_node_ptr()->is_dummy()); - - return (tmp); - } -}; - -template -bool operator==( const solist_iterator &i, const solist_iterator &j ) { - return i.my_node_ptr == j.my_node_ptr && i.my_list_ptr == j.my_list_ptr; -} -template -bool operator!=( const solist_iterator& i, const solist_iterator& j ) { - return i.my_node_ptr != j.my_node_ptr || i.my_list_ptr != j.my_list_ptr; -} - -// Forward type and class definitions -typedef size_t sokey_t; - - -// Forward list in which elements are sorted in a split-order -template -class split_ordered_list -{ -public: - typedef split_ordered_list self_type; - - typedef typename tbb::internal::allocator_rebind::type allocator_type; - - struct node; - typedef node *nodeptr_t; - - typedef typename tbb::internal::allocator_traits::value_type value_type; - typedef typename tbb::internal::allocator_traits::size_type size_type; - typedef typename tbb::internal::allocator_traits::difference_type difference_type; - typedef typename tbb::internal::allocator_traits::pointer pointer; - typedef typename tbb::internal::allocator_traits::const_pointer const_pointer; - // No support for reference/const_reference in allocator traits - typedef value_type& reference; - typedef const value_type& const_reference; - - typedef solist_iterator const_iterator; - typedef solist_iterator iterator; - typedef flist_iterator raw_const_iterator; - typedef flist_iterator raw_iterator; - - // Node that holds the element in a split-ordered list - struct node : tbb::internal::no_assign - { - private: - // for compilers that try to generate default constructors though they are not needed. - node(); // VS 2008, 2010, 2012 - public: - // Initialize the node with the given order key - void init(sokey_t order_key) { - my_order_key = order_key; - my_next = NULL; - } - - // Return the order key (needed for hashing) - sokey_t get_order_key() const { // TODO: remove - return my_order_key; - } - - // get() and value() is a common interface for getting access to node`s element (required by node_handle) - value_type* storage() { - return reinterpret_cast(&my_element); - } - - value_type& value() { - return *storage(); - } - - // Inserts the new element in the list in an atomic fashion - nodeptr_t atomic_set_next(nodeptr_t new_node, nodeptr_t current_node) - { - // Try to change the next pointer on the current element to a new element, only if it still points to the cached next - nodeptr_t exchange_node = tbb::internal::as_atomic(my_next).compare_and_swap(new_node, current_node); - - if (exchange_node == current_node) // TODO: why this branch? - { - // Operation succeeded, return the new node - return new_node; - } - else - { - // Operation failed, return the "interfering" node - return exchange_node; - } - } - - // Checks if this element in the list is a dummy, order enforcing node. Dummy nodes are used by buckets - // in the hash table to quickly index into the right subsection of the split-ordered list. - bool is_dummy() const { - return (my_order_key & 0x1) == 0; - } - - - nodeptr_t my_next; // Next element in the list - value_type my_element; // Element storage - sokey_t my_order_key; // Order key for this element - }; - - // Allocate a new node with the given order key; used to allocate dummy nodes - nodeptr_t create_node(sokey_t order_key) { - nodeptr_t pnode = my_node_allocator.allocate(1); - pnode->init(order_key); - return (pnode); - } - - // Allocate a new node with the given order key and value - template - nodeptr_t create_node(sokey_t order_key, __TBB_FORWARDING_REF(Arg) t, - /*AllowCreate=*/tbb::internal::true_type=tbb::internal::true_type()){ - nodeptr_t pnode = my_node_allocator.allocate(1); - - //TODO: use RAII scoped guard instead of explicit catch - __TBB_TRY { - new(static_cast(&pnode->my_element)) T(tbb::internal::forward(t)); - pnode->init(order_key); - } __TBB_CATCH(...) { - my_node_allocator.deallocate(pnode, 1); - __TBB_RETHROW(); - } - - return (pnode); - } - - // A helper to avoid excessive requiremens in internal_insert - template - nodeptr_t create_node(sokey_t, __TBB_FORWARDING_REF(Arg), - /*AllowCreate=*/tbb::internal::false_type){ - __TBB_ASSERT(false, "This compile-time helper should never get called"); - return nodeptr_t(); - } - - // Allocate a new node with the given parameters for constructing value - template - nodeptr_t create_node_v( __TBB_FORWARDING_REF(Args) __TBB_PARAMETER_PACK args){ - nodeptr_t pnode = my_node_allocator.allocate(1); - - //TODO: use RAII scoped guard instead of explicit catch - __TBB_TRY { - new(static_cast(&pnode->my_element)) T(__TBB_PACK_EXPANSION(tbb::internal::forward(args))); - } __TBB_CATCH(...) { - my_node_allocator.deallocate(pnode, 1); - __TBB_RETHROW(); - } - - return (pnode); - } - - split_ordered_list(allocator_type a = allocator_type()) - : my_node_allocator(a), my_element_count(0) - { - // Immediately allocate a dummy node with order key of 0. This node - // will always be the head of the list. - my_head = create_node(sokey_t(0)); - } - - ~split_ordered_list() - { - // Clear the list - clear(); - - // Remove the head element which is not cleared by clear() - nodeptr_t pnode = my_head; - my_head = NULL; - - __TBB_ASSERT(pnode != NULL && pnode->my_next == NULL, "Invalid head list node"); - - destroy_node(pnode); - } - - // Common forward list functions - - allocator_type get_allocator() const { - return (my_node_allocator); - } - - void clear() { - nodeptr_t pnext; - nodeptr_t pnode = my_head; - - __TBB_ASSERT(my_head != NULL, "Invalid head list node"); - pnext = pnode->my_next; - pnode->my_next = NULL; - pnode = pnext; - - while (pnode != NULL) - { - pnext = pnode->my_next; - destroy_node(pnode); - pnode = pnext; - } - - my_element_count = 0; - } - - // Returns a first non-dummy element in the SOL - iterator begin() { - return first_real_iterator(raw_begin()); - } - - // Returns a first non-dummy element in the SOL - const_iterator begin() const { - return first_real_iterator(raw_begin()); - } - - iterator end() { - return (iterator(0, this)); - } - - const_iterator end() const { - return (const_iterator(0, this)); - } - - const_iterator cbegin() const { - return (((const self_type *)this)->begin()); - } - - const_iterator cend() const { - return (((const self_type *)this)->end()); - } - - // Checks if the number of elements (non-dummy) is 0 - bool empty() const { - return (my_element_count == 0); - } - - // Returns the number of non-dummy elements in the list - size_type size() const { - return my_element_count; - } - - // Returns the maximum size of the list, determined by the allocator - size_type max_size() const { - return my_node_allocator.max_size(); - } - - // Swaps 'this' list with the passed in one - void swap(self_type& other) - { - if (this == &other) - { - // Nothing to do - return; - } - - std::swap(my_element_count, other.my_element_count); - std::swap(my_head, other.my_head); - } - - // Split-order list functions - - // Returns a first element in the SOL, which is always a dummy - raw_iterator raw_begin() { - return raw_iterator(my_head); - } - - // Returns a first element in the SOL, which is always a dummy - raw_const_iterator raw_begin() const { - return raw_const_iterator(my_head); - } - - raw_iterator raw_end() { - return raw_iterator(0); - } - - raw_const_iterator raw_end() const { - return raw_const_iterator(0); - } - - static sokey_t get_order_key(const raw_const_iterator& it) { - return it.get_node_ptr()->get_order_key(); - } - - static sokey_t get_safe_order_key(const raw_const_iterator& it) { - if( !it.get_node_ptr() ) return ~sokey_t(0); - return it.get_node_ptr()->get_order_key(); - } - - // Returns a public iterator version of the internal iterator. Public iterator must not - // be a dummy private iterator. - iterator get_iterator(raw_iterator it) { - __TBB_ASSERT(it.get_node_ptr() == NULL || !it.get_node_ptr()->is_dummy(), "Invalid user node (dummy)"); - return iterator(it.get_node_ptr(), this); - } - - // Returns a public iterator version of the internal iterator. Public iterator must not - // be a dummy private iterator. - const_iterator get_iterator(raw_const_iterator it) const { - __TBB_ASSERT(it.get_node_ptr() == NULL || !it.get_node_ptr()->is_dummy(), "Invalid user node (dummy)"); - return const_iterator(it.get_node_ptr(), this); - } - - // Returns a non-const version of the raw_iterator - raw_iterator get_iterator(raw_const_iterator it) { - return raw_iterator(it.get_node_ptr()); - } - - // Returns a non-const version of the iterator - static iterator get_iterator(const_iterator it) { - return iterator(it.my_node_ptr, it.my_list_ptr); - } - - // Returns a public iterator version of a first non-dummy internal iterator at or after - // the passed in internal iterator. - iterator first_real_iterator(raw_iterator it) - { - // Skip all dummy, internal only iterators - while (it != raw_end() && it.get_node_ptr()->is_dummy()) - ++it; - - return iterator(it.get_node_ptr(), this); - } - - // Returns a public iterator version of a first non-dummy internal iterator at or after - // the passed in internal iterator. - const_iterator first_real_iterator(raw_const_iterator it) const - { - // Skip all dummy, internal only iterators - while (it != raw_end() && it.get_node_ptr()->is_dummy()) - ++it; - - return const_iterator(it.get_node_ptr(), this); - } - - // Erase an element using the allocator - void destroy_node(nodeptr_t pnode) { - if (!pnode->is_dummy()) my_node_allocator.destroy(pnode); - my_node_allocator.deallocate(pnode, 1); - } - - // Try to insert a new element in the list. - // If insert fails, return the node that was inserted instead. - static nodeptr_t try_insert_atomic(nodeptr_t previous, nodeptr_t new_node, nodeptr_t current_node) { - new_node->my_next = current_node; - return previous->atomic_set_next(new_node, current_node); - } - - // Insert a new element between passed in iterators - std::pair try_insert(raw_iterator it, raw_iterator next, nodeptr_t pnode, size_type *new_count) - { - nodeptr_t inserted_node = try_insert_atomic(it.get_node_ptr(), pnode, next.get_node_ptr()); - - if (inserted_node == pnode) - { - // If the insert succeeded, check that the order is correct and increment the element count - check_range(it, next); - *new_count = tbb::internal::as_atomic(my_element_count).fetch_and_increment(); - return std::pair(iterator(pnode, this), true); - } - else - { - return std::pair(end(), false); - } - } - - // Insert a new dummy element, starting search at a parent dummy element - raw_iterator insert_dummy(raw_iterator it, sokey_t order_key) - { - raw_iterator last = raw_end(); - raw_iterator where = it; - - __TBB_ASSERT(where != last, "Invalid head node"); - - ++where; - - // Create a dummy element up front, even though it may be discarded (due to concurrent insertion) - nodeptr_t dummy_node = create_node(order_key); - - for (;;) - { - __TBB_ASSERT(it != last, "Invalid head list node"); - - // If the head iterator is at the end of the list, or past the point where this dummy - // node needs to be inserted, then try to insert it. - if (where == last || get_order_key(where) > order_key) - { - __TBB_ASSERT(get_order_key(it) < order_key, "Invalid node order in the list"); - - // Try to insert it in the right place - nodeptr_t inserted_node = try_insert_atomic(it.get_node_ptr(), dummy_node, where.get_node_ptr()); - - if (inserted_node == dummy_node) - { - // Insertion succeeded, check the list for order violations - check_range(it, where); - return raw_iterator(dummy_node); - } - else - { - // Insertion failed: either dummy node was inserted by another thread, or - // a real element was inserted at exactly the same place as dummy node. - // Proceed with the search from the previous location where order key was - // known to be larger (note: this is legal only because there is no safe - // concurrent erase operation supported). - where = it; - ++where; - continue; - } - } - else if (get_order_key(where) == order_key) - { - // Another dummy node with the same value found, discard the new one. - destroy_node(dummy_node); - return where; - } - - // Move the iterator forward - it = where; - ++where; - } - - } - - nodeptr_t erase_node_impl(raw_iterator previous, raw_const_iterator& where) { - nodeptr_t pnode = (where++).get_node_ptr(); - nodeptr_t prevnode = previous.get_node_ptr(); - __TBB_ASSERT(prevnode->my_next == pnode, "Erase must take consecutive iterators"); - prevnode->my_next = pnode->my_next; - return pnode; - } - - // This erase function can handle both real and dummy nodes - void erase_node(raw_iterator previous, raw_const_iterator& where, - /*allow_destroy*/tbb::internal::true_type) - { - nodeptr_t pnode = erase_node_impl(previous, where); - destroy_node(pnode); - } - - void erase_node(raw_iterator previous, raw_const_iterator& where, - /*allow_destroy*/tbb::internal::false_type) - { - erase_node_impl(previous, where); - } - - void erase_node(raw_iterator previous, raw_const_iterator& where) { - erase_node(previous, where, /*allow_destroy*/tbb::internal::true_type()); - } - - // Erase the element (previous node needs to be passed because this is a forward only list) - template - iterator erase_node(raw_iterator previous, const_iterator where, AllowDestroy) - { - raw_const_iterator it = where; - erase_node(previous, it, AllowDestroy()); - my_element_count--; - - return get_iterator(first_real_iterator(it)); - } - - iterator erase_node(raw_iterator previous, const_iterator& where) { - return erase_node(previous, where, /*allow_destroy*/tbb::internal::true_type()); - } - - - - // Move all elements from the passed in split-ordered list to this one - void move_all(self_type& source) - { - raw_const_iterator first = source.raw_begin(); - raw_const_iterator last = source.raw_end(); - - if (first == last) - return; - - nodeptr_t previous_node = my_head; - raw_const_iterator begin_iterator = first++; - - // Move all elements one by one, including dummy ones - for (raw_const_iterator it = first; it != last;) - { - nodeptr_t pnode = it.get_node_ptr(); - - nodeptr_t dummy_node = pnode->is_dummy() ? create_node(pnode->get_order_key()) : create_node(pnode->get_order_key(), pnode->my_element); - previous_node = try_insert_atomic(previous_node, dummy_node, NULL); - __TBB_ASSERT(previous_node != NULL, "Insertion must succeed"); - raw_const_iterator where = it++; - source.erase_node(get_iterator(begin_iterator), where); - } - check_range(); - } - - -private: - //Need to setup private fields of split_ordered_list in move constructor and assignment of concurrent_unordered_base - template - friend class concurrent_unordered_base; - - // Check the list for order violations - void check_range( raw_iterator first, raw_iterator last ) - { -#if TBB_USE_ASSERT - for (raw_iterator it = first; it != last; ++it) - { - raw_iterator next = it; - ++next; - - __TBB_ASSERT(next == raw_end() || get_order_key(next) >= get_order_key(it), "!!! List order inconsistency !!!"); - } -#else - tbb::internal::suppress_unused_warning(first, last); -#endif - } - void check_range() - { -#if TBB_USE_ASSERT - check_range( raw_begin(), raw_end() ); -#endif - } - - typename tbb::internal::allocator_rebind::type my_node_allocator; // allocator object for nodes - size_type my_element_count; // Total item count, not counting dummy nodes - nodeptr_t my_head; // pointer to head node -}; - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) -#pragma warning(push) -#pragma warning(disable: 4127) // warning C4127: conditional expression is constant -#endif - -template -class concurrent_unordered_base : public Traits -{ -protected: - // Type definitions - typedef concurrent_unordered_base self_type; - typedef typename Traits::value_type value_type; - typedef typename Traits::key_type key_type; - typedef typename Traits::hash_compare hash_compare; - typedef typename Traits::allocator_type allocator_type; - typedef typename hash_compare::hasher hasher; - typedef typename hash_compare::key_equal key_equal; - - typedef typename tbb::internal::allocator_traits::size_type size_type; - typedef typename tbb::internal::allocator_traits::difference_type difference_type; - typedef typename tbb::internal::allocator_traits::pointer pointer; - typedef typename tbb::internal::allocator_traits::const_pointer const_pointer; - // No support for reference/const_reference in allocator - typedef typename allocator_type::value_type& reference; - typedef const typename allocator_type::value_type& const_reference; - - typedef split_ordered_list solist_t; - typedef typename solist_t::nodeptr_t nodeptr_t; - // Iterators that walk the entire split-order list, including dummy nodes - typedef typename solist_t::raw_iterator raw_iterator; - typedef typename solist_t::raw_const_iterator raw_const_iterator; - typedef typename solist_t::iterator iterator; // TODO: restore const iterator for unordered_sets - typedef typename solist_t::const_iterator const_iterator; - typedef iterator local_iterator; - typedef const_iterator const_local_iterator; -#if __TBB_UNORDERED_NODE_HANDLE_PRESENT - typedef typename Traits::node_type node_type; -#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT - using Traits::my_hash_compare; - using Traits::get_key; - using Traits::allow_multimapping; - - static const size_type initial_bucket_number = 8; // Initial number of buckets - -private: - template - friend class concurrent_unordered_base; - - typedef std::pair pairii_t; - typedef std::pair paircc_t; - - static size_type const pointers_per_table = sizeof(size_type) * 8; // One bucket segment per bit - static const size_type initial_bucket_load = 4; // Initial maximum number of elements per bucket - - struct call_internal_clear_on_exit{ - concurrent_unordered_base* my_instance; - call_internal_clear_on_exit(concurrent_unordered_base* instance) : my_instance(instance) {} - void dismiss(){ my_instance = NULL;} - ~call_internal_clear_on_exit(){ - if (my_instance){ - my_instance->internal_clear(); - } - } - }; -protected: - // Constructors/Destructors - concurrent_unordered_base(size_type n_of_buckets = initial_bucket_number, - const hash_compare& hc = hash_compare(), const allocator_type& a = allocator_type()) - : Traits(hc), my_solist(a), - my_allocator(a), my_maximum_bucket_size((float) initial_bucket_load) - { - if( n_of_buckets == 0) ++n_of_buckets; - my_number_of_buckets = size_type(1)<<__TBB_Log2((uintptr_t)n_of_buckets*2-1); // round up to power of 2 - internal_init(); - } - - concurrent_unordered_base(const concurrent_unordered_base& right, const allocator_type& a) - : Traits(right.my_hash_compare), my_solist(a), my_allocator(a) - { - internal_init(); - internal_copy(right); - } - - concurrent_unordered_base(const concurrent_unordered_base& right) - : Traits(right.my_hash_compare), my_solist(right.get_allocator()), my_allocator(right.get_allocator()) - { - //FIXME:exception safety seems to be broken here - internal_init(); - internal_copy(right); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - concurrent_unordered_base(concurrent_unordered_base&& right) - : Traits(right.my_hash_compare), my_solist(right.get_allocator()), my_allocator(right.get_allocator()), - my_maximum_bucket_size(float(initial_bucket_load)) - { - my_number_of_buckets = initial_bucket_number; - internal_init(); - swap(right); - } - - concurrent_unordered_base(concurrent_unordered_base&& right, const allocator_type& a) - : Traits(right.my_hash_compare), my_solist(a), my_allocator(a) - { - call_internal_clear_on_exit clear_buckets_on_exception(this); - - internal_init(); - if (a == right.get_allocator()){ - my_number_of_buckets = initial_bucket_number; - my_maximum_bucket_size = float(initial_bucket_load); - this->swap(right); - }else{ - my_maximum_bucket_size = right.my_maximum_bucket_size; - my_number_of_buckets = right.my_number_of_buckets; - my_solist.my_element_count = right.my_solist.my_element_count; - - if (! right.my_solist.empty()){ - nodeptr_t previous_node = my_solist.my_head; - - // Move all elements one by one, including dummy ones - for (raw_const_iterator it = ++(right.my_solist.raw_begin()), last = right.my_solist.raw_end(); it != last; ++it) - { - const nodeptr_t pnode = it.get_node_ptr(); - nodeptr_t node; - if (pnode->is_dummy()) { - node = my_solist.create_node(pnode->get_order_key()); - size_type bucket = __TBB_ReverseBits(pnode->get_order_key()) % my_number_of_buckets; - set_bucket(bucket, node); - }else{ - node = my_solist.create_node(pnode->get_order_key(), std::move(pnode->my_element)); - } - - previous_node = my_solist.try_insert_atomic(previous_node, node, NULL); - __TBB_ASSERT(previous_node != NULL, "Insertion of node failed. Concurrent inserts in constructor ?"); - } - my_solist.check_range(); - } - } - - clear_buckets_on_exception.dismiss(); - } - -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - - concurrent_unordered_base& operator=(const concurrent_unordered_base& right) { - if (this != &right) - internal_copy(right); - return (*this); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - concurrent_unordered_base& operator=(concurrent_unordered_base&& other) - { - if(this != &other){ - typedef typename tbb::internal::allocator_traits::propagate_on_container_move_assignment pocma_t; - if(pocma_t::value || this->my_allocator == other.my_allocator) { - concurrent_unordered_base trash (std::move(*this)); - swap(other); - if (pocma_t::value) { - using std::swap; - //TODO: swapping allocators here may be a problem, replace with single direction moving - swap(this->my_solist.my_node_allocator, other.my_solist.my_node_allocator); - swap(this->my_allocator, other.my_allocator); - } - } else { - concurrent_unordered_base moved_copy(std::move(other),this->my_allocator); - this->swap(moved_copy); - } - } - return *this; - } - -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! assignment operator from initializer_list - concurrent_unordered_base& operator=(std::initializer_list il) - { - this->clear(); - this->insert(il.begin(),il.end()); - return (*this); - } -#endif // __TBB_INITIALIZER_LISTS_PRESENT - - - ~concurrent_unordered_base() { - // Delete all node segments - internal_clear(); - } - -#if __TBB_UNORDERED_NODE_HANDLE_PRESENT - template - void internal_merge(SourceType& source) { - typedef typename SourceType::iterator source_iterator; - __TBB_STATIC_ASSERT((tbb::internal::is_same_type::value), - "Incompatible containers cannot be merged"); - - for(source_iterator it = source.begin(); it != source.end();) { - source_iterator where = it++; - if (allow_multimapping || find(get_key(*where)) == end()) { - std::pair extract_result = source.internal_extract(where); - - // Remember the old order key - sokey_t old_order_key = extract_result.first.my_node->get_order_key(); - - // If the insertion fails, it returns ownership of the node to extract_result.first - // extract_result.first remains valid node handle - if (!insert(std::move(extract_result.first)).second) { - raw_iterator next = extract_result.second; - raw_iterator current = next++; - - // Revert order key to old value - extract_result.first.my_node->init(old_order_key); - - __TBB_ASSERT(extract_result.first.my_node->get_order_key() >= current.get_node_ptr()->get_order_key(), - "Wrong nodes order in source container"); - __TBB_ASSERT(next==source.my_solist.raw_end() || - extract_result.first.my_node->get_order_key() <= next.get_node_ptr()->get_order_key(), - "Wrong nodes order in source container"); - - size_t new_count = 0;// To use try_insert() - bool insert_result = - source.my_solist.try_insert(current, next, extract_result.first.my_node, &new_count).second; - __TBB_ASSERT_EX(insert_result, "Return to source must be successful. " - "Changing source container while merging is unsafe."); - } - extract_result.first.deactivate(); - } - } - } -#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT - -public: - allocator_type get_allocator() const { - return my_solist.get_allocator(); - } - - // Size and capacity function - bool empty() const { - return my_solist.empty(); - } - - size_type size() const { - return my_solist.size(); - } - - size_type max_size() const { - return my_solist.max_size(); - } - - // Iterators - iterator begin() { - return my_solist.begin(); - } - - const_iterator begin() const { - return my_solist.begin(); - } - - iterator end() { - return my_solist.end(); - } - - const_iterator end() const { - return my_solist.end(); - } - - const_iterator cbegin() const { - return my_solist.cbegin(); - } - - const_iterator cend() const { - return my_solist.cend(); - } - - // Parallel traversal support - class const_range_type : tbb::internal::no_assign { - const concurrent_unordered_base &my_table; - raw_const_iterator my_begin_node; - raw_const_iterator my_end_node; - mutable raw_const_iterator my_midpoint_node; - public: - //! Type for size of a range - typedef typename concurrent_unordered_base::size_type size_type; - typedef typename concurrent_unordered_base::value_type value_type; - typedef typename concurrent_unordered_base::reference reference; - typedef typename concurrent_unordered_base::difference_type difference_type; - typedef typename concurrent_unordered_base::const_iterator iterator; - - //! True if range is empty. - bool empty() const {return my_begin_node == my_end_node;} - - //! True if range can be partitioned into two subranges. - bool is_divisible() const { - return my_midpoint_node != my_end_node; - } - //! Split range. - const_range_type( const_range_type &r, split ) : - my_table(r.my_table), my_end_node(r.my_end_node) - { - r.my_end_node = my_begin_node = r.my_midpoint_node; - __TBB_ASSERT( !empty(), "Splitting despite the range is not divisible" ); - __TBB_ASSERT( !r.empty(), "Splitting despite the range is not divisible" ); - set_midpoint(); - r.set_midpoint(); - } - //! Init range with container and grainsize specified - const_range_type( const concurrent_unordered_base &a_table ) : - my_table(a_table), my_begin_node(a_table.my_solist.begin()), - my_end_node(a_table.my_solist.end()) - { - set_midpoint(); - } - iterator begin() const { return my_table.my_solist.get_iterator(my_begin_node); } - iterator end() const { return my_table.my_solist.get_iterator(my_end_node); } - //! The grain size for this range. - size_type grainsize() const { return 1; } - - //! Set my_midpoint_node to point approximately half way between my_begin_node and my_end_node. - void set_midpoint() const { - if( my_begin_node == my_end_node ) // not divisible - my_midpoint_node = my_end_node; - else { - sokey_t begin_key = solist_t::get_safe_order_key(my_begin_node); - sokey_t end_key = solist_t::get_safe_order_key(my_end_node); - size_t mid_bucket = __TBB_ReverseBits( begin_key + (end_key-begin_key)/2 ) % my_table.my_number_of_buckets; - while ( !my_table.is_initialized(mid_bucket) ) mid_bucket = my_table.get_parent(mid_bucket); - if(__TBB_ReverseBits(mid_bucket) > begin_key) { - // found a dummy_node between begin and end - my_midpoint_node = my_table.my_solist.first_real_iterator(my_table.get_bucket( mid_bucket )); - } - else { - // didn't find a dummy node between begin and end. - my_midpoint_node = my_end_node; - } -#if TBB_USE_ASSERT - { - sokey_t mid_key = solist_t::get_safe_order_key(my_midpoint_node); - __TBB_ASSERT( begin_key < mid_key, "my_begin_node is after my_midpoint_node" ); - __TBB_ASSERT( mid_key <= end_key, "my_midpoint_node is after my_end_node" ); - } -#endif // TBB_USE_ASSERT - } - } - }; - - class range_type : public const_range_type { - public: - typedef typename concurrent_unordered_base::iterator iterator; - //! Split range. - range_type( range_type &r, split ) : const_range_type( r, split() ) {} - //! Init range with container and grainsize specified - range_type( const concurrent_unordered_base &a_table ) : const_range_type(a_table) {} - - iterator begin() const { return solist_t::get_iterator( const_range_type::begin() ); } - iterator end() const { return solist_t::get_iterator( const_range_type::end() ); } - }; - - range_type range() { - return range_type( *this ); - } - - const_range_type range() const { - return const_range_type( *this ); - } - - // Modifiers - std::pair insert(const value_type& value) { - return internal_insert(value); - } - - iterator insert(const_iterator, const value_type& value) { - // Ignore hint - return insert(value).first; - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - std::pair insert(value_type&& value) { - return internal_insert(std::move(value)); - } - - iterator insert(const_iterator, value_type&& value) { - // Ignore hint - return insert(std::move(value)).first; - } -#endif /*__TBB_CPP11_RVALUE_REF_PRESENT*/ - -#if __TBB_UNORDERED_NODE_HANDLE_PRESENT - std::pair insert(node_type&& nh) { - if (!nh.empty()) { - nodeptr_t handled_node = nh.my_node; - std::pair insert_result = - internal_insert - (handled_node->my_element, handled_node); - if (insert_result.second) - nh.deactivate(); - return insert_result; - } - return std::pair(end(), false); - } - - iterator insert(const_iterator, node_type&& nh) { - return insert(std::move(nh)).first; - } -#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT - template - std::pair emplace(Args&&... args) { - nodeptr_t pnode = my_solist.create_node_v(tbb::internal::forward(args)...); - - return internal_insert(pnode->my_element, pnode); - } - - template - iterator emplace_hint(const_iterator, Args&&... args) { - // Ignore hint - return emplace(tbb::internal::forward(args)...).first; - } -#endif // __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT - - - template - void insert(Iterator first, Iterator last) { - for (Iterator it = first; it != last; ++it) - insert(*it); - } - -#if __TBB_INITIALIZER_LISTS_PRESENT - //! Insert initializer list - void insert(std::initializer_list il) { - insert(il.begin(), il.end()); - } -#endif - - iterator unsafe_erase(const_iterator where) { - return internal_erase(where); - } - - iterator unsafe_erase(const_iterator first, const_iterator last) { - while (first != last) - unsafe_erase(first++); - return my_solist.get_iterator(first); - } - - size_type unsafe_erase(const key_type& key) { - pairii_t where = equal_range(key); - size_type item_count = internal_distance(where.first, where.second); - unsafe_erase(where.first, where.second); - return item_count; - } - -#if __TBB_UNORDERED_NODE_HANDLE_PRESENT - node_type unsafe_extract(const_iterator where) { - return internal_extract(where).first; - } - - node_type unsafe_extract(const key_type& key) { - pairii_t where = equal_range(key); - if (where.first == end()) return node_type(); // element was not found - return internal_extract(where.first).first; - } -#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT - - void swap(concurrent_unordered_base& right) { - if (this != &right) { - std::swap(my_hash_compare, right.my_hash_compare); - my_solist.swap(right.my_solist); - internal_swap_buckets(right); - std::swap(my_number_of_buckets, right.my_number_of_buckets); - std::swap(my_maximum_bucket_size, right.my_maximum_bucket_size); - } - } - - // Observers - hasher hash_function() const { - return my_hash_compare.my_hash_object; - } - - key_equal key_eq() const { - return my_hash_compare.my_key_compare_object; - } - - void clear() { - // Clear list - my_solist.clear(); - - // Clear buckets - internal_clear(); - - // Initialize bucket 0 - __TBB_ASSERT(my_buckets[0] == NULL, NULL); - raw_iterator dummy_node = my_solist.raw_begin(); - set_bucket(0, dummy_node); - } - - // Lookup - iterator find(const key_type& key) { - return internal_find(key); - } - - const_iterator find(const key_type& key) const { - return const_cast(this)->internal_find(key); - } - - size_type count(const key_type& key) const { - if(allow_multimapping) { - paircc_t answer = equal_range(key); - size_type item_count = internal_distance(answer.first, answer.second); - return item_count; - } else { - return const_cast(this)->internal_find(key) == end()?0:1; - } - } - - std::pair equal_range(const key_type& key) { - return internal_equal_range(key); - } - - std::pair equal_range(const key_type& key) const { - return const_cast(this)->internal_equal_range(key); - } - - // Bucket interface - for debugging - size_type unsafe_bucket_count() const { - return my_number_of_buckets; - } - - size_type unsafe_max_bucket_count() const { - return segment_size(pointers_per_table-1); - } - - size_type unsafe_bucket_size(size_type bucket) { - size_type item_count = 0; - if (is_initialized(bucket)) { - raw_iterator it = get_bucket(bucket); - ++it; - for (; it != my_solist.raw_end() && !it.get_node_ptr()->is_dummy(); ++it) - ++item_count; - } - return item_count; - } - - size_type unsafe_bucket(const key_type& key) const { - sokey_t order_key = (sokey_t) my_hash_compare(key); - size_type bucket = order_key % my_number_of_buckets; - return bucket; - } - - // If the bucket is initialized, return a first non-dummy element in it - local_iterator unsafe_begin(size_type bucket) { - if (!is_initialized(bucket)) - return end(); - - raw_iterator it = get_bucket(bucket); - return my_solist.first_real_iterator(it); - } - - // If the bucket is initialized, return a first non-dummy element in it - const_local_iterator unsafe_begin(size_type bucket) const - { - if (!is_initialized(bucket)) - return end(); - - raw_const_iterator it = get_bucket(bucket); - return my_solist.first_real_iterator(it); - } - - // @REVIEW: Takes O(n) - // Returns the iterator after the last non-dummy element in the bucket - local_iterator unsafe_end(size_type bucket) - { - if (!is_initialized(bucket)) - return end(); - - raw_iterator it = get_bucket(bucket); - - // Find the end of the bucket, denoted by the dummy element - do ++it; - while(it != my_solist.raw_end() && !it.get_node_ptr()->is_dummy()); - - // Return the first real element past the end of the bucket - return my_solist.first_real_iterator(it); - } - - // @REVIEW: Takes O(n) - // Returns the iterator after the last non-dummy element in the bucket - const_local_iterator unsafe_end(size_type bucket) const - { - if (!is_initialized(bucket)) - return end(); - - raw_const_iterator it = get_bucket(bucket); - - // Find the end of the bucket, denoted by the dummy element - do ++it; - while(it != my_solist.raw_end() && !it.get_node_ptr()->is_dummy()); - - // Return the first real element past the end of the bucket - return my_solist.first_real_iterator(it); - } - - const_local_iterator unsafe_cbegin(size_type bucket) const { - return ((const self_type *) this)->unsafe_begin(bucket); - } - - const_local_iterator unsafe_cend(size_type bucket) const { - return ((const self_type *) this)->unsafe_end(bucket); - } - - // Hash policy - float load_factor() const { - return (float) size() / (float) unsafe_bucket_count(); - } - - float max_load_factor() const { - return my_maximum_bucket_size; - } - - void max_load_factor(float newmax) { - if (newmax != newmax || newmax < 0) - tbb::internal::throw_exception(tbb::internal::eid_invalid_load_factor); - my_maximum_bucket_size = newmax; - } - - // This function is a noop, because the underlying split-ordered list - // is already sorted, so an increase in the bucket number will be - // reflected next time this bucket is touched. - void rehash(size_type buckets) { - size_type current_buckets = my_number_of_buckets; - if (current_buckets >= buckets) - return; - my_number_of_buckets = size_type(1)<<__TBB_Log2((uintptr_t)buckets*2-1); // round up to power of 2 - } - -private: - - // Initialize the hash and keep the first bucket open - void internal_init() { - // Initialize the array of segment pointers - memset(my_buckets, 0, sizeof(my_buckets)); - - // Initialize bucket 0 - raw_iterator dummy_node = my_solist.raw_begin(); - set_bucket(0, dummy_node); - } - - void internal_clear() { - for (size_type index = 0; index < pointers_per_table; ++index) { - if (my_buckets[index] != NULL) { - size_type sz = segment_size(index); - for (size_type index2 = 0; index2 < sz; ++index2) - my_allocator.destroy(&my_buckets[index][index2]); - my_allocator.deallocate(my_buckets[index], sz); - my_buckets[index] = 0; - } - } - } - - void internal_copy(const self_type& right) { - clear(); - - my_maximum_bucket_size = right.my_maximum_bucket_size; - my_number_of_buckets = right.my_number_of_buckets; - - __TBB_TRY { - insert(right.begin(), right.end()); - my_hash_compare = right.my_hash_compare; - } __TBB_CATCH(...) { - my_solist.clear(); - __TBB_RETHROW(); - } - } - - void internal_swap_buckets(concurrent_unordered_base& right) - { - // Swap all node segments - for (size_type index = 0; index < pointers_per_table; ++index) - { - raw_iterator * iterator_pointer = my_buckets[index]; - my_buckets[index] = right.my_buckets[index]; - right.my_buckets[index] = iterator_pointer; - } - } - - //TODO: why not use std::distance? - // Hash APIs - static size_type internal_distance(const_iterator first, const_iterator last) - { - size_type num = 0; - - for (const_iterator it = first; it != last; ++it) - ++num; - - return num; - } - - // Insert an element in the hash given its value - template - std::pair internal_insert(__TBB_FORWARDING_REF(ValueType) value, nodeptr_t pnode = NULL) - { - const key_type *pkey = &get_key(value); - sokey_t hash_key = (sokey_t) my_hash_compare(*pkey); - size_type new_count = 0; - sokey_t order_key = split_order_key_regular(hash_key); - raw_iterator previous = prepare_bucket(hash_key); - raw_iterator last = my_solist.raw_end(); - __TBB_ASSERT(previous != last, "Invalid head node"); - - if (pnode) { - // Set new order_key to node - pnode->init(order_key); - } - - // First node is a dummy node - for (raw_iterator where = previous;;) - { - ++where; - if (where == last || solist_t::get_order_key(where) > order_key || - // if multimapped, stop at the first item equal to us. - (allow_multimapping && solist_t::get_order_key(where) == order_key && - !my_hash_compare(get_key(*where), *pkey))) // TODO: fix negation - { - if (!pnode) { - pnode = my_solist.create_node(order_key, tbb::internal::forward(value), AllowCreate()); - // If the value was moved, the known reference to key might be invalid - pkey = &get_key(pnode->my_element); - } - - // Try to insert 'pnode' between 'previous' and 'where' - std::pair result = my_solist.try_insert(previous, where, pnode, &new_count); - - if (result.second) - { - // Insertion succeeded, adjust the table size, if needed - adjust_table_size(new_count, my_number_of_buckets); - return result; - } - else - { - // Insertion failed: either the same node was inserted by another thread, or - // another element was inserted at exactly the same place as this node. - // Proceed with the search from the previous location where order key was - // known to be larger (note: this is legal only because there is no safe - // concurrent erase operation supported). - where = previous; - continue; - } - } - else if (!allow_multimapping && solist_t::get_order_key(where) == order_key && - !my_hash_compare(get_key(*where), *pkey)) // TODO: fix negation - { // Element already in the list, return it - if (pnode && AllowDestroy::value) - my_solist.destroy_node(pnode); - return std::pair(my_solist.get_iterator(where), false); - } - // Move the iterator forward - previous = where; - } - } - - // Find the element in the split-ordered list - iterator internal_find(const key_type& key) - { - sokey_t hash_key = (sokey_t) my_hash_compare(key); - sokey_t order_key = split_order_key_regular(hash_key); - raw_iterator last = my_solist.raw_end(); - - for (raw_iterator it = prepare_bucket(hash_key); it != last; ++it) - { - if (solist_t::get_order_key(it) > order_key) - { - // If the order key is smaller than the current order key, the element - // is not in the hash. - return end(); - } - else if (solist_t::get_order_key(it) == order_key) - { - // The fact that order keys match does not mean that the element is found. - // Key function comparison has to be performed to check whether this is the - // right element. If not, keep searching while order key is the same. - if (!my_hash_compare(get_key(*it), key)) // TODO: fix negation - return my_solist.get_iterator(it); - } - } - - return end(); - } - - // Erase an element from the list. This is not a concurrency safe function. - iterator internal_erase(const_iterator it) - { - sokey_t hash_key = (sokey_t) my_hash_compare(get_key(*it)); - raw_iterator previous = prepare_bucket(hash_key); - raw_iterator last = my_solist.raw_end(); - __TBB_ASSERT(previous != last, "Invalid head node"); - - // First node is a dummy node - for (raw_iterator where = previous; where != last; previous = where) { - ++where; - if (my_solist.get_iterator(where) == it) - return my_solist.erase_node(previous, it); - } - return end(); - } - -#if __TBB_UNORDERED_NODE_HANDLE_PRESENT - std::pair internal_extract(const_iterator it) { - sokey_t hash_key = sokey_t(my_hash_compare(get_key(*it))); - raw_iterator previous = prepare_bucket(hash_key); - raw_iterator last = my_solist.raw_end(); - __TBB_ASSERT(previous != last, "Invalid head node"); - - for(raw_iterator where = previous; where != last; previous = where) { - ++where; - if (my_solist.get_iterator(where) == it) { - const_iterator result = it; - my_solist.erase_node(previous, it, /*allow_destroy*/tbb::internal::false_type()); - return std::pair( node_type(result.get_node_ptr()), - previous); - } - } - return std::pair(node_type(), end()); - } -#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT - - // Return the [begin, end) pair of iterators with the same key values. - // This operation makes sense only if mapping is many-to-one. - pairii_t internal_equal_range(const key_type& key) - { - sokey_t hash_key = (sokey_t) my_hash_compare(key); - sokey_t order_key = split_order_key_regular(hash_key); - raw_iterator end_it = my_solist.raw_end(); - - for (raw_iterator it = prepare_bucket(hash_key); it != end_it; ++it) - { - if (solist_t::get_order_key(it) > order_key) - { - // There is no element with the given key - return pairii_t(end(), end()); - } - else if (solist_t::get_order_key(it) == order_key && - !my_hash_compare(get_key(*it), key)) // TODO: fix negation; also below - { - iterator first = my_solist.get_iterator(it); - iterator last = first; - do ++last; while( allow_multimapping && last != end() && !my_hash_compare(get_key(*last), key) ); - return pairii_t(first, last); - } - } - - return pairii_t(end(), end()); - } - - // Bucket APIs - void init_bucket(size_type bucket) - { - // Bucket 0 has no parent. - __TBB_ASSERT( bucket != 0, "The first bucket must always be initialized"); - - size_type parent_bucket = get_parent(bucket); - - // All parent_bucket buckets have to be initialized before this bucket is - if (!is_initialized(parent_bucket)) - init_bucket(parent_bucket); - - raw_iterator parent = get_bucket(parent_bucket); - - // Create a dummy first node in this bucket - raw_iterator dummy_node = my_solist.insert_dummy(parent, split_order_key_dummy(bucket)); - set_bucket(bucket, dummy_node); - } - - void adjust_table_size(size_type total_elements, size_type current_size) - { - // Grow the table by a factor of 2 if possible and needed - if ( ((float) total_elements / (float) current_size) > my_maximum_bucket_size ) - { - // Double the size of the hash only if size has not changed in between loads - my_number_of_buckets.compare_and_swap(2u*current_size, current_size); - //Simple "my_number_of_buckets.compare_and_swap( current_size<<1, current_size );" does not work for VC8 - //due to overzealous compiler warnings in /Wp64 mode - } - } - - size_type get_parent(size_type bucket) const - { - // Unsets bucket's most significant turned-on bit - size_type msb = __TBB_Log2((uintptr_t)bucket); - return bucket & ~(size_type(1) << msb); - } - - - // Dynamic sized array (segments) - //! @return segment index of given index in the array - static size_type segment_index_of( size_type index ) { - return size_type( __TBB_Log2( uintptr_t(index|1) ) ); - } - - //! @return the first array index of given segment - static size_type segment_base( size_type k ) { - return (size_type(1)<(new_segment), 0, sz*sizeof(raw_iterator)); - - if (my_buckets[segment].compare_and_swap( new_segment, NULL) != NULL) - my_allocator.deallocate(new_segment, sz); - } - - my_buckets[segment][bucket] = dummy_head; - } - - bool is_initialized(size_type bucket) const { - size_type segment = segment_index_of(bucket); - bucket -= segment_base(segment); - - if (my_buckets[segment] == NULL) - return false; - - raw_iterator it = my_buckets[segment][bucket]; - return (it.get_node_ptr() != NULL); - } - - // Utilities for keys - - // A regular order key has its original hash value reversed and the last bit set - sokey_t split_order_key_regular(sokey_t order_key) const { - return __TBB_ReverseBits(order_key) | 0x1; - } - - // A dummy order key has its original hash value reversed and the last bit unset - sokey_t split_order_key_dummy(sokey_t order_key) const { - return __TBB_ReverseBits(order_key) & ~sokey_t(0x1); - } - - // Shared variables - atomic my_number_of_buckets; // Current table size - solist_t my_solist; // List where all the elements are kept - typename tbb::internal::allocator_rebind::type my_allocator; // Allocator object for segments - float my_maximum_bucket_size; // Maximum size of the bucket - atomic my_buckets[pointers_per_table]; // The segment table -}; -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) -#pragma warning(pop) // warning 4127 is back -#endif - -} // namespace internal -//! @endcond -} // namespace interface5 -} // namespace tbb -#endif // __TBB__concurrent_unordered_impl_H diff --git a/src/3rd party/tbb/internal/_deprecated_header_message_guard.h b/src/3rd party/tbb/internal/_deprecated_header_message_guard.h deleted file mode 100644 index a92982090a..0000000000 --- a/src/3rd party/tbb/internal/_deprecated_header_message_guard.h +++ /dev/null @@ -1,69 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include "tbb/tbb_config.h" - -#if (!defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) || (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0)) && !defined(__TBB_INTERNAL_INCLUDES_DEPRECATION_MESSAGE) && \ -!defined(__TBB_condition_variable_H_include_area) && \ -!defined(__TBB_ppl_H_include_area) && \ -!defined(__TBB_thread_H_include_area) && \ -!defined(__TBB_tuple_H_include_area) && \ -!defined(__TBB_aggregator_H_include_area) && \ -!defined(__TBB_aligned_space_H_include_area) && \ -!defined(__TBB_atomic_H_include_area) && \ -!defined(__TBB_combinable_H_include_area) && \ -!defined(__TBB_concurrent_hash_map_H_include_area) && \ -!defined(__TBB_concurrent_lru_cache_H_include_area) && \ -!defined(__TBB_concurrent_map_H_include_area) && \ -!defined(__TBB_concurrent_priority_queue_H_include_area) && \ -!defined(__TBB_concurrent_queue_H_include_area) && \ -!defined(__TBB_concurrent_set_H_include_area) && \ -!defined(__TBB_concurrent_unordered_map_H_include_area) && \ -!defined(__TBB_concurrent_unordered_set_H_include_area) && \ -!defined(__TBB_concurrent_vector_H_include_area) && \ -!defined(__TBB_critical_section_H_include_area) && \ -!defined(__TBB_enumerable_thread_specific_H_include_area) && \ -!defined(__TBB_flow_graph_opencl_node_H_include_area) && \ -!defined(__TBB_flow_graph_H_include_area) && \ -!defined(__TBB_mutex_H_include_area) && \ -!defined(__TBB_parallel_do_H_include_area) && \ -!defined(__TBB_parallel_for_H_include_area) && \ -!defined(__TBB_parallel_invoke_H_include_area) && \ -!defined(__TBB_parallel_reduce_H_include_area) && \ -!defined(__TBB_parallel_scan_H_include_area) && \ -!defined(__TBB_parallel_sort_H_include_area) && \ -!defined(__TBB_parallel_while_H_include_area) && \ -!defined(__TBB_partitioner_H_include_area) && \ -!defined(__TBB_pipeline_H_include_area) && \ -!defined(__TBB_queuing_mutex_H_include_area) && \ -!defined(__TBB_queuing_rw_mutex_H_include_area) && \ -!defined(__TBB_reader_writer_lock_H_include_area) && \ -!defined(__TBB_recursive_mutex_H_include_area) && \ -!defined(__TBB_runtime_loader_H_include_area) && \ -!defined(__TBB_task_scheduler_init_H_include_area) && \ -!defined(__TBB_spin_mutex_H_include_area) && \ -!defined(__TBB_task_arena_H_include_area) && \ -!defined(__TBB_task_group_H_include_area) && \ -!defined(__TBB_task_scheduler_observer_H_include_area) && \ -!defined(__TBB_task_H_include_area) && \ -!defined(__TBB_tbb_exception_H_include_area) && \ -!defined(__TBB_tbb_profiling_H_include_area) && \ -!defined(__TBB_tbb_thread_H_include_area) && \ -!defined(__TBB_tbb_H_include_area) - -#define __TBB_show_deprecated_header_message - -#endif diff --git a/src/3rd party/tbb/internal/_flow_graph_async_msg_impl.h b/src/3rd party/tbb/internal/_flow_graph_async_msg_impl.h deleted file mode 100644 index ffb63ce4fe..0000000000 --- a/src/3rd party/tbb/internal/_flow_graph_async_msg_impl.h +++ /dev/null @@ -1,153 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB__flow_graph_async_msg_impl_H -#define __TBB__flow_graph_async_msg_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -namespace internal { - -template -class async_storage { -public: - typedef receiver async_storage_client; - - async_storage() : my_graph(nullptr) { - my_data_ready.store(false); - } - - ~async_storage() { - // Release reference to the graph if async_storage - // was destructed before set() call - if (my_graph) { - my_graph->release_wait(); - my_graph = nullptr; - } - } - - template - async_storage(C&& data) : my_graph(nullptr), my_data( std::forward(data) ) { - using namespace tbb::internal; - __TBB_STATIC_ASSERT( (is_same_type::type, typename strip::type>::value), "incoming type must be T" ); - - my_data_ready.store(true); - } - - template - bool set(C&& data) { - using namespace tbb::internal; - __TBB_STATIC_ASSERT( (is_same_type::type, typename strip::type>::value), "incoming type must be T" ); - - { - tbb::spin_mutex::scoped_lock locker(my_mutex); - - if (my_data_ready.load()) { - __TBB_ASSERT(false, "double set() call"); - return false; - } - - my_data = std::forward(data); - my_data_ready.store(true); - } - - // Thread sync is on my_data_ready flag - for (typename subscriber_list_type::iterator it = my_clients.begin(); it != my_clients.end(); ++it) { - (*it)->try_put(my_data); - } - - // Data was sent, release reference to the graph - if (my_graph) { - my_graph->release_wait(); - my_graph = nullptr; - } - - return true; - } - - task* subscribe(async_storage_client& client, graph& g) { - if (! my_data_ready.load()) - { - tbb::spin_mutex::scoped_lock locker(my_mutex); - - if (! my_data_ready.load()) { -#if TBB_USE_ASSERT - for (typename subscriber_list_type::iterator it = my_clients.begin(); it != my_clients.end(); ++it) { - __TBB_ASSERT(*it != &client, "unexpected double subscription"); - } -#endif // TBB_USE_ASSERT - - // Increase graph lifetime - my_graph = &g; - my_graph->reserve_wait(); - - // Subscribe - my_clients.push_back(&client); - return SUCCESSFULLY_ENQUEUED; - } - } - - __TBB_ASSERT(my_data_ready.load(), "data is NOT ready"); - return client.try_put_task(my_data); - } - -private: - graph* my_graph; - tbb::spin_mutex my_mutex; - tbb::atomic my_data_ready; - T my_data; - typedef std::vector subscriber_list_type; - subscriber_list_type my_clients; -}; - -} // namespace internal - -template -class async_msg { - template< typename > friend class receiver; - template< typename, typename > friend struct internal::async_helpers; -public: - typedef T async_msg_data_type; - - async_msg() : my_storage(std::make_shared< internal::async_storage >()) {} - - async_msg(const T& t) : my_storage(std::make_shared< internal::async_storage >(t)) {} - - async_msg(T&& t) : my_storage(std::make_shared< internal::async_storage >( std::move(t) )) {} - - virtual ~async_msg() {} - - void set(const T& t) { - my_storage->set(t); - } - - void set(T&& t) { - my_storage->set( std::move(t) ); - } - -protected: - // Can be overridden in derived class to inform that - // async calculation chain is over - virtual void finalize() const {} - -private: - typedef std::shared_ptr< internal::async_storage > async_storage_ptr; - async_storage_ptr my_storage; -}; - -#endif // __TBB__flow_graph_async_msg_impl_H diff --git a/src/3rd party/tbb/internal/_flow_graph_cache_impl.h b/src/3rd party/tbb/internal/_flow_graph_cache_impl.h deleted file mode 100644 index 65297715c9..0000000000 --- a/src/3rd party/tbb/internal/_flow_graph_cache_impl.h +++ /dev/null @@ -1,592 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB__flow_graph_cache_impl_H -#define __TBB__flow_graph_cache_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -// included in namespace tbb::flow::interfaceX (in flow_graph.h) - -namespace internal { - -//! A node_cache maintains a std::queue of elements of type T. Each operation is protected by a lock. -template< typename T, typename M=spin_mutex > -class node_cache { - public: - - typedef size_t size_type; - - bool empty() { - typename mutex_type::scoped_lock lock( my_mutex ); - return internal_empty(); - } - - void add( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - internal_push(n); - } - - void remove( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - for ( size_t i = internal_size(); i != 0; --i ) { - T &s = internal_pop(); - if ( &s == &n ) return; // only remove one predecessor per request - internal_push(s); - } - } - - void clear() { - while( !my_q.empty()) (void)my_q.pop(); -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - my_built_predecessors.clear(); -#endif - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef edge_container built_predecessors_type; - built_predecessors_type &built_predecessors() { return my_built_predecessors; } - - typedef typename edge_container::edge_list_type predecessor_list_type; - void internal_add_built_predecessor( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - my_built_predecessors.add_edge(n); - } - - void internal_delete_built_predecessor( T &n ) { - typename mutex_type::scoped_lock lock( my_mutex ); - my_built_predecessors.delete_edge(n); - } - - void copy_predecessors( predecessor_list_type &v) { - typename mutex_type::scoped_lock lock( my_mutex ); - my_built_predecessors.copy_edges(v); - } - - size_t predecessor_count() { - typename mutex_type::scoped_lock lock(my_mutex); - return (size_t)(my_built_predecessors.edge_count()); - } -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - -protected: - - typedef M mutex_type; - mutex_type my_mutex; - std::queue< T * > my_q; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - built_predecessors_type my_built_predecessors; -#endif - - // Assumes lock is held - inline bool internal_empty( ) { - return my_q.empty(); - } - - // Assumes lock is held - inline size_type internal_size( ) { - return my_q.size(); - } - - // Assumes lock is held - inline void internal_push( T &n ) { - my_q.push(&n); - } - - // Assumes lock is held - inline T &internal_pop() { - T *v = my_q.front(); - my_q.pop(); - return *v; - } - -}; - -//! A cache of predecessors that only supports try_get -template< typename T, typename M=spin_mutex > -#if __TBB_PREVIEW_ASYNC_MSG -// TODO: make predecessor_cache type T-independent when async_msg becomes regular feature -class predecessor_cache : public node_cache< untyped_sender, M > { -#else -class predecessor_cache : public node_cache< sender, M > { -#endif // __TBB_PREVIEW_ASYNC_MSG -public: - typedef M mutex_type; - typedef T output_type; -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_sender predecessor_type; - typedef untyped_receiver successor_type; -#else - typedef sender predecessor_type; - typedef receiver successor_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - - predecessor_cache( ) : my_owner( NULL ) { } - - void set_owner( successor_type *owner ) { my_owner = owner; } - - bool get_item( output_type &v ) { - - bool msg = false; - - do { - predecessor_type *src; - { - typename mutex_type::scoped_lock lock(this->my_mutex); - if ( this->internal_empty() ) { - break; - } - src = &this->internal_pop(); - } - - // Try to get from this sender - msg = src->try_get( v ); - - if (msg == false) { - // Relinquish ownership of the edge - if (my_owner) - src->register_successor( *my_owner ); - } else { - // Retain ownership of the edge - this->add(*src); - } - } while ( msg == false ); - return msg; - } - - // If we are removing arcs (rf_clear_edges), call clear() rather than reset(). - void reset() { - if (my_owner) { - for(;;) { - predecessor_type *src; - { - if (this->internal_empty()) break; - src = &this->internal_pop(); - } - src->register_successor( *my_owner ); - } - } - } - -protected: - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - using node_cache< predecessor_type, M >::my_built_predecessors; -#endif - successor_type *my_owner; -}; - -//! An cache of predecessors that supports requests and reservations -// TODO: make reservable_predecessor_cache type T-independent when async_msg becomes regular feature -template< typename T, typename M=spin_mutex > -class reservable_predecessor_cache : public predecessor_cache< T, M > { -public: - typedef M mutex_type; - typedef T output_type; -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_sender predecessor_type; - typedef untyped_receiver successor_type; -#else - typedef sender predecessor_type; - typedef receiver successor_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - - reservable_predecessor_cache( ) : reserved_src(NULL) { } - - bool - try_reserve( output_type &v ) { - bool msg = false; - - do { - { - typename mutex_type::scoped_lock lock(this->my_mutex); - if ( reserved_src || this->internal_empty() ) - return false; - - reserved_src = &this->internal_pop(); - } - - // Try to get from this sender - msg = reserved_src->try_reserve( v ); - - if (msg == false) { - typename mutex_type::scoped_lock lock(this->my_mutex); - // Relinquish ownership of the edge - reserved_src->register_successor( *this->my_owner ); - reserved_src = NULL; - } else { - // Retain ownership of the edge - this->add( *reserved_src ); - } - } while ( msg == false ); - - return msg; - } - - bool - try_release( ) { - reserved_src->try_release( ); - reserved_src = NULL; - return true; - } - - bool - try_consume( ) { - reserved_src->try_consume( ); - reserved_src = NULL; - return true; - } - - void reset( ) { - reserved_src = NULL; - predecessor_cache::reset( ); - } - - void clear() { - reserved_src = NULL; - predecessor_cache::clear(); - } - -private: - predecessor_type *reserved_src; -}; - - -//! An abstract cache of successors -// TODO: make successor_cache type T-independent when async_msg becomes regular feature -template -class successor_cache : tbb::internal::no_copy { -protected: - - typedef M mutex_type; - mutex_type my_mutex; - -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_receiver successor_type; - typedef untyped_receiver *pointer_type; - typedef untyped_sender owner_type; -#else - typedef receiver successor_type; - typedef receiver *pointer_type; - typedef sender owner_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - typedef std::list< pointer_type > successors_type; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - edge_container my_built_successors; -#endif - successors_type my_successors; - - owner_type *my_owner; - -public: -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename edge_container::edge_list_type successor_list_type; - - edge_container &built_successors() { return my_built_successors; } - - void internal_add_built_successor( successor_type &r) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.add_edge( r ); - } - - void internal_delete_built_successor( successor_type &r) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.delete_edge(r); - } - - void copy_successors( successor_list_type &v) { - typename mutex_type::scoped_lock l(my_mutex, false); - my_built_successors.copy_edges(v); - } - - size_t successor_count() { - typename mutex_type::scoped_lock l(my_mutex,false); - return my_built_successors.edge_count(); - } - -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - - successor_cache( ) : my_owner(NULL) {} - - void set_owner( owner_type *owner ) { my_owner = owner; } - - virtual ~successor_cache() {} - - void register_successor( successor_type &r ) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_successors.push_back( &r ); - } - - void remove_successor( successor_type &r ) { - typename mutex_type::scoped_lock l(my_mutex, true); - for ( typename successors_type::iterator i = my_successors.begin(); - i != my_successors.end(); ++i ) { - if ( *i == & r ) { - my_successors.erase(i); - break; - } - } - } - - bool empty() { - typename mutex_type::scoped_lock l(my_mutex, false); - return my_successors.empty(); - } - - void clear() { - my_successors.clear(); -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - my_built_successors.clear(); -#endif - } - -#if !__TBB_PREVIEW_ASYNC_MSG - virtual task * try_put_task( const T &t ) = 0; -#endif // __TBB_PREVIEW_ASYNC_MSG - }; // successor_cache - -//! An abstract cache of successors, specialized to continue_msg -template -class successor_cache< continue_msg, M > : tbb::internal::no_copy { -protected: - - typedef M mutex_type; - mutex_type my_mutex; - -#if __TBB_PREVIEW_ASYNC_MSG - typedef untyped_receiver successor_type; - typedef untyped_receiver *pointer_type; -#else - typedef receiver successor_type; - typedef receiver *pointer_type; -#endif // __TBB_PREVIEW_ASYNC_MSG - typedef std::list< pointer_type > successors_type; - successors_type my_successors; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - edge_container my_built_successors; - typedef edge_container::edge_list_type successor_list_type; -#endif - - sender *my_owner; - -public: - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - - edge_container &built_successors() { return my_built_successors; } - - void internal_add_built_successor( successor_type &r) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.add_edge( r ); - } - - void internal_delete_built_successor( successor_type &r) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_built_successors.delete_edge(r); - } - - void copy_successors( successor_list_type &v) { - typename mutex_type::scoped_lock l(my_mutex, false); - my_built_successors.copy_edges(v); - } - - size_t successor_count() { - typename mutex_type::scoped_lock l(my_mutex,false); - return my_built_successors.edge_count(); - } - -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - - successor_cache( ) : my_owner(NULL) {} - - void set_owner( sender *owner ) { my_owner = owner; } - - virtual ~successor_cache() {} - - void register_successor( successor_type &r ) { - typename mutex_type::scoped_lock l(my_mutex, true); - my_successors.push_back( &r ); - if ( my_owner && r.is_continue_receiver() ) { - r.register_predecessor( *my_owner ); - } - } - - void remove_successor( successor_type &r ) { - typename mutex_type::scoped_lock l(my_mutex, true); - for ( successors_type::iterator i = my_successors.begin(); - i != my_successors.end(); ++i ) { - if ( *i == & r ) { - // TODO: Check if we need to test for continue_receiver before - // removing from r. - if ( my_owner ) - r.remove_predecessor( *my_owner ); - my_successors.erase(i); - break; - } - } - } - - bool empty() { - typename mutex_type::scoped_lock l(my_mutex, false); - return my_successors.empty(); - } - - void clear() { - my_successors.clear(); -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - my_built_successors.clear(); -#endif - } - -#if !__TBB_PREVIEW_ASYNC_MSG - virtual task * try_put_task( const continue_msg &t ) = 0; -#endif // __TBB_PREVIEW_ASYNC_MSG - -}; // successor_cache< continue_msg > - -//! A cache of successors that are broadcast to -// TODO: make broadcast_cache type T-independent when async_msg becomes regular feature -template -class broadcast_cache : public successor_cache { - typedef M mutex_type; - typedef typename successor_cache::successors_type successors_type; - -public: - - broadcast_cache( ) {} - - // as above, but call try_put_task instead, and return the last task we received (if any) -#if __TBB_PREVIEW_ASYNC_MSG - template - task * try_put_task( const X &t ) { -#else - task * try_put_task( const T &t ) __TBB_override { -#endif // __TBB_PREVIEW_ASYNC_MSG - task * last_task = NULL; - bool upgraded = true; - typename mutex_type::scoped_lock l(this->my_mutex, upgraded); - typename successors_type::iterator i = this->my_successors.begin(); - while ( i != this->my_successors.end() ) { - task *new_task = (*i)->try_put_task(t); - // workaround for icc bug - graph& graph_ref = (*i)->graph_reference(); - last_task = combine_tasks(graph_ref, last_task, new_task); // enqueue if necessary - if(new_task) { - ++i; - } - else { // failed - if ( (*i)->register_predecessor(*this->my_owner) ) { - if (!upgraded) { - l.upgrade_to_writer(); - upgraded = true; - } - i = this->my_successors.erase(i); - } else { - ++i; - } - } - } - return last_task; - } - - // call try_put_task and return list of received tasks -#if __TBB_PREVIEW_ASYNC_MSG - template - bool gather_successful_try_puts( const X &t, task_list &tasks ) { -#else - bool gather_successful_try_puts( const T &t, task_list &tasks ) { -#endif // __TBB_PREVIEW_ASYNC_MSG - bool upgraded = true; - bool is_at_least_one_put_successful = false; - typename mutex_type::scoped_lock l(this->my_mutex, upgraded); - typename successors_type::iterator i = this->my_successors.begin(); - while ( i != this->my_successors.end() ) { - task * new_task = (*i)->try_put_task(t); - if(new_task) { - ++i; - if(new_task != SUCCESSFULLY_ENQUEUED) { - tasks.push_back(*new_task); - } - is_at_least_one_put_successful = true; - } - else { // failed - if ( (*i)->register_predecessor(*this->my_owner) ) { - if (!upgraded) { - l.upgrade_to_writer(); - upgraded = true; - } - i = this->my_successors.erase(i); - } else { - ++i; - } - } - } - return is_at_least_one_put_successful; - } -}; - -//! A cache of successors that are put in a round-robin fashion -// TODO: make round_robin_cache type T-independent when async_msg becomes regular feature -template -class round_robin_cache : public successor_cache { - typedef size_t size_type; - typedef M mutex_type; - typedef typename successor_cache::successors_type successors_type; - -public: - - round_robin_cache( ) {} - - size_type size() { - typename mutex_type::scoped_lock l(this->my_mutex, false); - return this->my_successors.size(); - } - -#if __TBB_PREVIEW_ASYNC_MSG - template - task * try_put_task( const X &t ) { -#else - task *try_put_task( const T &t ) __TBB_override { -#endif // __TBB_PREVIEW_ASYNC_MSG - bool upgraded = true; - typename mutex_type::scoped_lock l(this->my_mutex, upgraded); - typename successors_type::iterator i = this->my_successors.begin(); - while ( i != this->my_successors.end() ) { - task *new_task = (*i)->try_put_task(t); - if ( new_task ) { - return new_task; - } else { - if ( (*i)->register_predecessor(*this->my_owner) ) { - if (!upgraded) { - l.upgrade_to_writer(); - upgraded = true; - } - i = this->my_successors.erase(i); - } - else { - ++i; - } - } - } - return NULL; - } -}; - -} // namespace internal - -#endif // __TBB__flow_graph_cache_impl_H diff --git a/src/3rd party/tbb/internal/_flow_graph_impl.h b/src/3rd party/tbb/internal/_flow_graph_impl.h deleted file mode 100644 index e2356e2e36..0000000000 --- a/src/3rd party/tbb/internal/_flow_graph_impl.h +++ /dev/null @@ -1,547 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_flow_graph_impl_H -#define __TBB_flow_graph_impl_H - -#include "../tbb_stddef.h" -#include "../task.h" -#include "../task_arena.h" -#include "../flow_graph_abstractions.h" - -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES -#include "../concurrent_priority_queue.h" -#endif - -#include - -#if TBB_DEPRECATED_FLOW_ENQUEUE -#define FLOW_SPAWN(a) tbb::task::enqueue((a)) -#else -#define FLOW_SPAWN(a) tbb::task::spawn((a)) -#endif - -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES -#define __TBB_FLOW_GRAPH_PRIORITY_EXPR( expr ) expr -#define __TBB_FLOW_GRAPH_PRIORITY_ARG0( priority ) , priority -#define __TBB_FLOW_GRAPH_PRIORITY_ARG1( arg1, priority ) arg1, priority -#else -#define __TBB_FLOW_GRAPH_PRIORITY_EXPR( expr ) -#define __TBB_FLOW_GRAPH_PRIORITY_ARG0( priority ) -#define __TBB_FLOW_GRAPH_PRIORITY_ARG1( arg1, priority ) arg1 -#endif // __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES - -#if TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR -#define __TBB_DEPRECATED_LIMITER_EXPR( expr ) expr -#define __TBB_DEPRECATED_LIMITER_ARG2( arg1, arg2 ) arg1, arg2 -#define __TBB_DEPRECATED_LIMITER_ARG4( arg1, arg2, arg3, arg4 ) arg1, arg3, arg4 -#else -#define __TBB_DEPRECATED_LIMITER_EXPR( expr ) -#define __TBB_DEPRECATED_LIMITER_ARG2( arg1, arg2 ) arg1 -#define __TBB_DEPRECATED_LIMITER_ARG4( arg1, arg2, arg3, arg4 ) arg1, arg2 -#endif // TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR - -namespace tbb { -namespace flow { - -namespace internal { -static tbb::task * const SUCCESSFULLY_ENQUEUED = (task *)-1; -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES -typedef unsigned int node_priority_t; -static const node_priority_t no_priority = node_priority_t(0); -#endif -} - -namespace interface10 { -class graph; -} - -namespace interface11 { - -using tbb::flow::internal::SUCCESSFULLY_ENQUEUED; - -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES -using tbb::flow::internal::node_priority_t; -using tbb::flow::internal::no_priority; -//! Base class for tasks generated by graph nodes. -struct graph_task : public task { - graph_task( node_priority_t node_priority = no_priority ) : priority( node_priority ) {} - node_priority_t priority; -}; -#else -typedef task graph_task; -#endif /* __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES */ - -class graph_node; - -template -class graph_iterator { - friend class tbb::flow::interface10::graph; - friend class graph_node; -public: - typedef size_t size_type; - typedef GraphNodeType value_type; - typedef GraphNodeType* pointer; - typedef GraphNodeType& reference; - typedef const GraphNodeType& const_reference; - typedef std::forward_iterator_tag iterator_category; - - //! Default constructor - graph_iterator() : my_graph(NULL), current_node(NULL) {} - - //! Copy constructor - graph_iterator(const graph_iterator& other) : - my_graph(other.my_graph), current_node(other.current_node) - {} - - //! Assignment - graph_iterator& operator=(const graph_iterator& other) { - if (this != &other) { - my_graph = other.my_graph; - current_node = other.current_node; - } - return *this; - } - - //! Dereference - reference operator*() const; - - //! Dereference - pointer operator->() const; - - //! Equality - bool operator==(const graph_iterator& other) const { - return ((my_graph == other.my_graph) && (current_node == other.current_node)); - } - - //! Inequality - bool operator!=(const graph_iterator& other) const { return !(operator==(other)); } - - //! Pre-increment - graph_iterator& operator++() { - internal_forward(); - return *this; - } - - //! Post-increment - graph_iterator operator++(int) { - graph_iterator result = *this; - operator++(); - return result; - } - -private: - // the graph over which we are iterating - GraphContainerType *my_graph; - // pointer into my_graph's my_nodes list - pointer current_node; - - //! Private initializing constructor for begin() and end() iterators - graph_iterator(GraphContainerType *g, bool begin); - void internal_forward(); -}; // class graph_iterator - -// flags to modify the behavior of the graph reset(). Can be combined. -enum reset_flags { - rf_reset_protocol = 0, - rf_reset_bodies = 1 << 0, // delete the current node body, reset to a copy of the initial node body. - rf_clear_edges = 1 << 1 // delete edges -}; - -namespace internal { - -void activate_graph(tbb::flow::interface10::graph& g); -void deactivate_graph(tbb::flow::interface10::graph& g); -bool is_graph_active(tbb::flow::interface10::graph& g); -tbb::task& prioritize_task(tbb::flow::interface10::graph& g, tbb::task& arena_task); -void spawn_in_graph_arena(tbb::flow::interface10::graph& g, tbb::task& arena_task); -void enqueue_in_graph_arena(tbb::flow::interface10::graph &g, tbb::task& arena_task); -void add_task_to_graph_reset_list(tbb::flow::interface10::graph& g, tbb::task *tp); - -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES -struct graph_task_comparator { - bool operator()(const graph_task* left, const graph_task* right) { - return left->priority < right->priority; - } -}; - -typedef tbb::concurrent_priority_queue graph_task_priority_queue_t; - -class priority_task_selector : public task { -public: - priority_task_selector(graph_task_priority_queue_t& priority_queue) - : my_priority_queue(priority_queue) {} - task* execute() __TBB_override { - graph_task* t = NULL; - bool result = my_priority_queue.try_pop(t); - __TBB_ASSERT_EX( result, "Number of critical tasks for scheduler and tasks" - " in graph's priority queue mismatched" ); - __TBB_ASSERT( t && t != SUCCESSFULLY_ENQUEUED, - "Incorrect task submitted to graph priority queue" ); - __TBB_ASSERT( t->priority != tbb::flow::internal::no_priority, - "Tasks from graph's priority queue must have priority" ); - task* t_next = t->execute(); - task::destroy(*t); - return t_next; - } -private: - graph_task_priority_queue_t& my_priority_queue; -}; -#endif /* __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES */ - -} - -} // namespace interfaceX -namespace interface10 { -//! The graph class -/** This class serves as a handle to the graph */ -class graph : tbb::internal::no_copy, public tbb::flow::graph_proxy { - friend class tbb::flow::interface11::graph_node; - - template< typename Body > - class run_task : public tbb::flow::interface11::graph_task { - public: - run_task(Body& body -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES - , tbb::flow::interface11::node_priority_t node_priority = tbb::flow::interface11::no_priority - ) : tbb::flow::interface11::graph_task(node_priority), -#else - ) : -#endif - my_body(body) { } - tbb::task *execute() __TBB_override { - my_body(); - return NULL; - } - private: - Body my_body; - }; - - template< typename Receiver, typename Body > - class run_and_put_task : public tbb::flow::interface11::graph_task { - public: - run_and_put_task(Receiver &r, Body& body) : my_receiver(r), my_body(body) {} - tbb::task *execute() __TBB_override { - tbb::task *res = my_receiver.try_put_task(my_body()); - if (res == tbb::flow::interface11::SUCCESSFULLY_ENQUEUED) res = NULL; - return res; - } - private: - Receiver &my_receiver; - Body my_body; - }; - typedef std::list task_list_type; - - class wait_functor { - tbb::task* graph_root_task; - public: - wait_functor(tbb::task* t) : graph_root_task(t) {} - void operator()() const { graph_root_task->wait_for_all(); } - }; - - //! A functor that spawns a task - class spawn_functor : tbb::internal::no_assign { - tbb::task& spawn_task; - public: - spawn_functor(tbb::task& t) : spawn_task(t) {} - void operator()() const { - FLOW_SPAWN(spawn_task); - } - }; - - void prepare_task_arena(bool reinit = false) { - if (reinit) { - __TBB_ASSERT(my_task_arena, "task arena is NULL"); - my_task_arena->terminate(); - my_task_arena->initialize(tbb::task_arena::attach()); - } - else { - __TBB_ASSERT(my_task_arena == NULL, "task arena is not NULL"); - my_task_arena = new tbb::task_arena(tbb::task_arena::attach()); - } - if (!my_task_arena->is_active()) // failed to attach - my_task_arena->initialize(); // create a new, default-initialized arena - __TBB_ASSERT(my_task_arena->is_active(), "task arena is not active"); - } - -public: - //! Constructs a graph with isolated task_group_context - graph(); - - //! Constructs a graph with use_this_context as context - explicit graph(tbb::task_group_context& use_this_context); - - //! Destroys the graph. - /** Calls wait_for_all, then destroys the root task and context. */ - ~graph(); - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - void set_name(const char *name); -#endif - - void increment_wait_count() { - reserve_wait(); - } - - void decrement_wait_count() { - release_wait(); - } - - //! Used to register that an external entity may still interact with the graph. - /** The graph will not return from wait_for_all until a matching number of decrement_wait_count calls - is made. */ - void reserve_wait() __TBB_override; - - //! Deregisters an external entity that may have interacted with the graph. - /** The graph will not return from wait_for_all until all the number of decrement_wait_count calls - matches the number of increment_wait_count calls. */ - void release_wait() __TBB_override; - - //! Spawns a task that runs a body and puts its output to a specific receiver - /** The task is spawned as a child of the graph. This is useful for running tasks - that need to block a wait_for_all() on the graph. For example a one-off source. */ - template< typename Receiver, typename Body > - void run(Receiver &r, Body body) { - if (tbb::flow::interface11::internal::is_graph_active(*this)) { - task* rtask = new (task::allocate_additional_child_of(*root_task())) - run_and_put_task< Receiver, Body >(r, body); - my_task_arena->execute(spawn_functor(*rtask)); - } - } - - //! Spawns a task that runs a function object - /** The task is spawned as a child of the graph. This is useful for running tasks - that need to block a wait_for_all() on the graph. For example a one-off source. */ - template< typename Body > - void run(Body body) { - if (tbb::flow::interface11::internal::is_graph_active(*this)) { - task* rtask = new (task::allocate_additional_child_of(*root_task())) run_task< Body >(body); - my_task_arena->execute(spawn_functor(*rtask)); - } - } - - //! Wait until graph is idle and decrement_wait_count calls equals increment_wait_count calls. - /** The waiting thread will go off and steal work while it is block in the wait_for_all. */ - void wait_for_all() { - cancelled = false; - caught_exception = false; - if (my_root_task) { -#if TBB_USE_EXCEPTIONS - try { -#endif - my_task_arena->execute(wait_functor(my_root_task)); -#if __TBB_TASK_GROUP_CONTEXT - cancelled = my_context->is_group_execution_cancelled(); -#endif -#if TBB_USE_EXCEPTIONS - } - catch (...) { - my_root_task->set_ref_count(1); - my_context->reset(); - caught_exception = true; - cancelled = true; - throw; - } -#endif -#if __TBB_TASK_GROUP_CONTEXT - // TODO: the "if" condition below is just a work-around to support the concurrent wait - // mode. The cancellation and exception mechanisms are still broken in this mode. - // Consider using task group not to re-implement the same functionality. - if (!(my_context->traits() & tbb::task_group_context::concurrent_wait)) { - my_context->reset(); // consistent with behavior in catch() -#endif - my_root_task->set_ref_count(1); -#if __TBB_TASK_GROUP_CONTEXT - } -#endif - } - } - - //! Returns the root task of the graph - tbb::task * root_task() { - return my_root_task; - } - - // ITERATORS - template - friend class tbb::flow::interface11::graph_iterator; - - // Graph iterator typedefs - typedef tbb::flow::interface11::graph_iterator iterator; - typedef tbb::flow::interface11::graph_iterator const_iterator; - - // Graph iterator constructors - //! start iterator - iterator begin(); - //! end iterator - iterator end(); - //! start const iterator - const_iterator begin() const; - //! end const iterator - const_iterator end() const; - //! start const iterator - const_iterator cbegin() const; - //! end const iterator - const_iterator cend() const; - - //! return status of graph execution - bool is_cancelled() { return cancelled; } - bool exception_thrown() { return caught_exception; } - - // thread-unsafe state reset. - void reset(tbb::flow::interface11::reset_flags f = tbb::flow::interface11::rf_reset_protocol); - -private: - tbb::task *my_root_task; -#if __TBB_TASK_GROUP_CONTEXT - tbb::task_group_context *my_context; -#endif - bool own_context; - bool cancelled; - bool caught_exception; - bool my_is_active; - task_list_type my_reset_task_list; - - tbb::flow::interface11::graph_node *my_nodes, *my_nodes_last; - - tbb::spin_mutex nodelist_mutex; - void register_node(tbb::flow::interface11::graph_node *n); - void remove_node(tbb::flow::interface11::graph_node *n); - - tbb::task_arena* my_task_arena; - -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES - tbb::flow::interface11::internal::graph_task_priority_queue_t my_priority_queue; -#endif - - friend void tbb::flow::interface11::internal::activate_graph(graph& g); - friend void tbb::flow::interface11::internal::deactivate_graph(graph& g); - friend bool tbb::flow::interface11::internal::is_graph_active(graph& g); - friend tbb::task& tbb::flow::interface11::internal::prioritize_task(graph& g, tbb::task& arena_task); - friend void tbb::flow::interface11::internal::spawn_in_graph_arena(graph& g, tbb::task& arena_task); - friend void tbb::flow::interface11::internal::enqueue_in_graph_arena(graph &g, tbb::task& arena_task); - friend void tbb::flow::interface11::internal::add_task_to_graph_reset_list(graph& g, tbb::task *tp); - - friend class tbb::interface7::internal::task_arena_base; - -}; // class graph -} // namespace interface10 - -namespace interface11 { - -using tbb::flow::interface10::graph; - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET -namespace internal{ -class get_graph_helper; -} -#endif - -//! The base of all graph nodes. -class graph_node : tbb::internal::no_copy { - friend class graph; - template - friend class graph_iterator; - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - friend class internal::get_graph_helper; -#endif - -protected: - graph& my_graph; - graph_node *next, *prev; -public: - explicit graph_node(graph& g); - - virtual ~graph_node(); - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - virtual void set_name(const char *name) = 0; -#endif - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - virtual void extract() = 0; -#endif - -protected: - // performs the reset on an individual node. - virtual void reset_node(reset_flags f = rf_reset_protocol) = 0; -}; // class graph_node - -namespace internal { - -inline void activate_graph(graph& g) { - g.my_is_active = true; -} - -inline void deactivate_graph(graph& g) { - g.my_is_active = false; -} - -inline bool is_graph_active(graph& g) { - return g.my_is_active; -} - -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES -inline tbb::task& prioritize_task(graph& g, tbb::task& t) { - task* critical_task = &t; - // TODO: change flow graph's interfaces to work with graph_task type instead of tbb::task. - graph_task* gt = static_cast(&t); - if( gt->priority != no_priority ) { - //! Non-preemptive priority pattern. The original task is submitted as a work item to the - //! priority queue, and a new critical task is created to take and execute a work item with - //! the highest known priority. The reference counting responsibility is transferred (via - //! allocate_continuation) to the new task. - critical_task = new( gt->allocate_continuation() ) priority_task_selector(g.my_priority_queue); - tbb::internal::make_critical( *critical_task ); - g.my_priority_queue.push(gt); - } - return *critical_task; -} -#else -inline tbb::task& prioritize_task(graph&, tbb::task& t) { - return t; -} -#endif /* __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES */ - -//! Spawns a task inside graph arena -inline void spawn_in_graph_arena(graph& g, tbb::task& arena_task) { - if (is_graph_active(g)) { - graph::spawn_functor s_fn(prioritize_task(g, arena_task)); - __TBB_ASSERT(g.my_task_arena && g.my_task_arena->is_active(), NULL); - g.my_task_arena->execute(s_fn); - } -} - -//! Enqueues a task inside graph arena -inline void enqueue_in_graph_arena(graph &g, tbb::task& arena_task) { - if (is_graph_active(g)) { - __TBB_ASSERT( g.my_task_arena && g.my_task_arena->is_active(), "Is graph's arena initialized and active?" ); - task::enqueue(prioritize_task(g, arena_task), *g.my_task_arena); - } -} - -inline void add_task_to_graph_reset_list(graph& g, tbb::task *tp) { - g.my_reset_task_list.push_back(tp); -} - -} // namespace internal - -} // namespace interfaceX -} // namespace flow -} // namespace tbb - -#endif // __TBB_flow_graph_impl_H diff --git a/src/3rd party/tbb/internal/_flow_graph_indexer_impl.h b/src/3rd party/tbb/internal/_flow_graph_indexer_impl.h deleted file mode 100644 index dc674f6c1b..0000000000 --- a/src/3rd party/tbb/internal/_flow_graph_indexer_impl.h +++ /dev/null @@ -1,480 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB__flow_graph_indexer_impl_H -#define __TBB__flow_graph_indexer_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include "_flow_graph_types_impl.h" - -namespace internal { - - // Output of the indexer_node is a tbb::flow::tagged_msg, and will be of - // the form tagged_msg - // where the value of tag will indicate which result was put to the - // successor. - - template - task* do_try_put(const T &v, void *p) { - typename IndexerNodeBaseType::output_type o(K, v); - return reinterpret_cast(p)->try_put_task(&o); - } - - template - struct indexer_helper { - template - static inline void set_indexer_node_pointer(PortTuple &my_input, IndexerNodeBaseType *p, graph& g) { - typedef typename tuple_element::type T; - task *(*indexer_node_put_task)(const T&, void *) = do_try_put; - tbb::flow::get(my_input).set_up(p, indexer_node_put_task, g); - indexer_helper::template set_indexer_node_pointer(my_input, p, g); - } - template - static inline void reset_inputs(InputTuple &my_input, reset_flags f) { - indexer_helper::reset_inputs(my_input, f); - tbb::flow::get(my_input).reset_receiver(f); - } -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - template - static inline void extract(InputTuple &my_input) { - indexer_helper::extract(my_input); - tbb::flow::get(my_input).extract_receiver(); - } -#endif - }; - - template - struct indexer_helper { - template - static inline void set_indexer_node_pointer(PortTuple &my_input, IndexerNodeBaseType *p, graph& g) { - typedef typename tuple_element<0, TupleTypes>::type T; - task *(*indexer_node_put_task)(const T&, void *) = do_try_put; - tbb::flow::get<0>(my_input).set_up(p, indexer_node_put_task, g); - } - template - static inline void reset_inputs(InputTuple &my_input, reset_flags f) { - tbb::flow::get<0>(my_input).reset_receiver(f); - } -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - template - static inline void extract(InputTuple &my_input) { - tbb::flow::get<0>(my_input).extract_receiver(); - } -#endif - }; - - template - class indexer_input_port : public receiver { - private: - void* my_indexer_ptr; - typedef task* (* forward_function_ptr)(T const &, void* ); - forward_function_ptr my_try_put_task; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - spin_mutex my_pred_mutex; - typedef typename receiver::built_predecessors_type built_predecessors_type; - built_predecessors_type my_built_predecessors; -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - graph* my_graph; - public: -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - indexer_input_port() : my_pred_mutex(), my_graph(NULL) {} - indexer_input_port( const indexer_input_port & other) : receiver(), my_pred_mutex(), my_graph(other.my_graph) { - } -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - void set_up(void* p, forward_function_ptr f, graph& g) { - my_indexer_ptr = p; - my_try_put_task = f; - my_graph = &g; - } -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename receiver::predecessor_type predecessor_type; - - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - size_t predecessor_count() __TBB_override { - spin_mutex::scoped_lock l(my_pred_mutex); - return my_built_predecessors.edge_count(); - } - void internal_add_built_predecessor(predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l(my_pred_mutex); - my_built_predecessors.add_edge(p); - } - void internal_delete_built_predecessor(predecessor_type &p) __TBB_override { - spin_mutex::scoped_lock l(my_pred_mutex); - my_built_predecessors.delete_edge(p); - } - void copy_predecessors( predecessor_list_type &v) __TBB_override { - spin_mutex::scoped_lock l(my_pred_mutex); - my_built_predecessors.copy_edges(v); - } -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task *try_put_task(const T &v) __TBB_override { - return my_try_put_task(v, my_indexer_ptr); - } - - graph& graph_reference() const __TBB_override { - return *my_graph; - } - - public: -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - void reset_receiver(reset_flags f) __TBB_override { if(f&rf_clear_edges) my_built_predecessors.clear(); } -#else - void reset_receiver(reset_flags /*f*/) __TBB_override { } -#endif - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - void extract_receiver() { my_built_predecessors.receiver_extract(*this); } -#endif - }; - - template - class indexer_node_FE { - public: - static const int N = tbb::flow::tuple_size::value; - typedef OutputType output_type; - typedef InputTuple input_type; - - // Some versions of Intel(R) C++ Compiler fail to generate an implicit constructor for the class which has std::tuple as a member. - indexer_node_FE() : my_inputs() {} - - input_type &input_ports() { return my_inputs; } - protected: - input_type my_inputs; - }; - - //! indexer_node_base - template - class indexer_node_base : public graph_node, public indexer_node_FE, - public sender { - protected: - using graph_node::my_graph; - public: - static const size_t N = tbb::flow::tuple_size::value; - typedef OutputType output_type; - typedef StructTypes tuple_types; - typedef typename sender::successor_type successor_type; - typedef indexer_node_FE input_ports_type; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename sender::built_successors_type built_successors_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - - private: - // ----------- Aggregator ------------ - enum op_type { reg_succ, rem_succ, try__put_task -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - , add_blt_succ, del_blt_succ, - blt_succ_cnt, blt_succ_cpy -#endif - }; - typedef indexer_node_base class_type; - - class indexer_node_base_operation : public aggregated_operation { - public: - char type; - union { - output_type const *my_arg; - successor_type *my_succ; - task *bypass_t; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - size_t cnt_val; - successor_list_type *succv; -#endif - }; - indexer_node_base_operation(const output_type* e, op_type t) : - type(char(t)), my_arg(e) {} - indexer_node_base_operation(const successor_type &s, op_type t) : type(char(t)), - my_succ(const_cast(&s)) {} - indexer_node_base_operation(op_type t) : type(char(t)) {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator my_aggregator; - - void handle_operations(indexer_node_base_operation* op_list) { - indexer_node_base_operation *current; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - - case reg_succ: - my_successors.register_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - - case rem_succ: - my_successors.remove_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case try__put_task: { - current->bypass_t = my_successors.try_put_task(*(current->my_arg)); - __TBB_store_with_release(current->status, SUCCEEDED); // return of try_put_task actual return value - } - break; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - case add_blt_succ: - my_successors.internal_add_built_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case del_blt_succ: - my_successors.internal_delete_built_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_succ_cnt: - current->cnt_val = my_successors.successor_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_succ_cpy: - my_successors.copy_successors(*(current->succv)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - } - } - } - // ---------- end aggregator ----------- - public: - indexer_node_base(graph& g) : graph_node(g), input_ports_type() { - indexer_helper::set_indexer_node_pointer(this->my_inputs, this, g); - my_successors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - indexer_node_base(const indexer_node_base& other) : graph_node(other.my_graph), input_ports_type(), sender() { - indexer_helper::set_indexer_node_pointer(this->my_inputs, this, other.my_graph); - my_successors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - bool register_successor(successor_type &r) __TBB_override { - indexer_node_base_operation op_data(r, reg_succ); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - bool remove_successor( successor_type &r) __TBB_override { - indexer_node_base_operation op_data(r, rem_succ); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - task * try_put_task(output_type const *v) { // not a virtual method in this class - indexer_node_base_operation op_data(v, try__put_task); - my_aggregator.execute(&op_data); - return op_data.bypass_t; - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - void internal_add_built_successor( successor_type &r) __TBB_override { - indexer_node_base_operation op_data(r, add_blt_succ); - my_aggregator.execute(&op_data); - } - - void internal_delete_built_successor( successor_type &r) __TBB_override { - indexer_node_base_operation op_data(r, del_blt_succ); - my_aggregator.execute(&op_data); - } - - size_t successor_count() __TBB_override { - indexer_node_base_operation op_data(blt_succ_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_successors( successor_list_type &v) __TBB_override { - indexer_node_base_operation op_data(blt_succ_cpy); - op_data.succv = &v; - my_aggregator.execute(&op_data); - } - void extract() __TBB_override { - my_successors.built_successors().sender_extract(*this); - indexer_helper::extract(this->my_inputs); - } -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - protected: - void reset_node(reset_flags f) __TBB_override { - if(f & rf_clear_edges) { - my_successors.clear(); - indexer_helper::reset_inputs(this->my_inputs,f); - } - } - - private: - broadcast_cache my_successors; - }; //indexer_node_base - - - template struct input_types; - - template - struct input_types<1, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<2, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<3, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<4, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<5, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<6, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename tuple_element<5, InputTuple>::type sixth_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<7, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename tuple_element<5, InputTuple>::type sixth_type; - typedef typename tuple_element<6, InputTuple>::type seventh_type; - typedef typename internal::tagged_msg type; - }; - - - template - struct input_types<8, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename tuple_element<5, InputTuple>::type sixth_type; - typedef typename tuple_element<6, InputTuple>::type seventh_type; - typedef typename tuple_element<7, InputTuple>::type eighth_type; - typedef typename internal::tagged_msg type; - }; - - - template - struct input_types<9, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename tuple_element<5, InputTuple>::type sixth_type; - typedef typename tuple_element<6, InputTuple>::type seventh_type; - typedef typename tuple_element<7, InputTuple>::type eighth_type; - typedef typename tuple_element<8, InputTuple>::type nineth_type; - typedef typename internal::tagged_msg type; - }; - - template - struct input_types<10, InputTuple> { - typedef typename tuple_element<0, InputTuple>::type first_type; - typedef typename tuple_element<1, InputTuple>::type second_type; - typedef typename tuple_element<2, InputTuple>::type third_type; - typedef typename tuple_element<3, InputTuple>::type fourth_type; - typedef typename tuple_element<4, InputTuple>::type fifth_type; - typedef typename tuple_element<5, InputTuple>::type sixth_type; - typedef typename tuple_element<6, InputTuple>::type seventh_type; - typedef typename tuple_element<7, InputTuple>::type eighth_type; - typedef typename tuple_element<8, InputTuple>::type nineth_type; - typedef typename tuple_element<9, InputTuple>::type tenth_type; - typedef typename internal::tagged_msg type; - }; - - // type generators - template - struct indexer_types : public input_types::value, OutputTuple> { - static const int N = tbb::flow::tuple_size::value; - typedef typename input_types::type output_type; - typedef typename wrap_tuple_elements::type input_ports_type; - typedef internal::indexer_node_FE indexer_FE_type; - typedef internal::indexer_node_base indexer_base_type; - }; - - template - class unfolded_indexer_node : public indexer_types::indexer_base_type { - public: - typedef typename indexer_types::input_ports_type input_ports_type; - typedef OutputTuple tuple_types; - typedef typename indexer_types::output_type output_type; - private: - typedef typename indexer_types::indexer_base_type base_type; - public: - unfolded_indexer_node(graph& g) : base_type(g) {} - unfolded_indexer_node(const unfolded_indexer_node &other) : base_type(other) {} - }; - -} /* namespace internal */ - -#endif /* __TBB__flow_graph_indexer_impl_H */ diff --git a/src/3rd party/tbb/internal/_flow_graph_item_buffer_impl.h b/src/3rd party/tbb/internal/_flow_graph_item_buffer_impl.h deleted file mode 100644 index 9ac4dbbbf4..0000000000 --- a/src/3rd party/tbb/internal/_flow_graph_item_buffer_impl.h +++ /dev/null @@ -1,284 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB__flow_graph_item_buffer_impl_H -#define __TBB__flow_graph_item_buffer_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include "tbb/internal/_flow_graph_types_impl.h" // for aligned_pair - -// in namespace tbb::flow::interfaceX (included in _flow_graph_node_impl.h) - - //! Expandable buffer of items. The possible operations are push, pop, - //* tests for empty and so forth. No mutual exclusion is built in. - //* objects are constructed into and explicitly-destroyed. get_my_item gives - // a read-only reference to the item in the buffer. set_my_item may be called - // with either an empty or occupied slot. - - using internal::aligned_pair; - using internal::alignment_of; - -namespace internal { - - template > - class item_buffer { - public: - typedef T item_type; - enum buffer_item_state { no_item=0, has_item=1, reserved_item=2 }; - protected: - typedef size_t size_type; - typedef typename aligned_pair::type buffer_item_type; - typedef typename A::template rebind::other allocator_type; - - buffer_item_type *my_array; - size_type my_array_size; - static const size_type initial_buffer_size = 4; - size_type my_head; - size_type my_tail; - - bool buffer_empty() const { return my_head == my_tail; } - - buffer_item_type &item(size_type i) { - __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].second))%alignment_of::value),NULL); - __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].first))%alignment_of::value), NULL); - return my_array[i & (my_array_size - 1) ]; - } - - const buffer_item_type &item(size_type i) const { - __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].second))%alignment_of::value), NULL); - __TBB_ASSERT(!(size_type(&(my_array[i&(my_array_size-1)].first))%alignment_of::value), NULL); - return my_array[i & (my_array_size-1)]; - } - - bool my_item_valid(size_type i) const { return (i < my_tail) && (i >= my_head) && (item(i).second != no_item); } - bool my_item_reserved(size_type i) const { return item(i).second == reserved_item; } - - // object management in buffer - const item_type &get_my_item(size_t i) const { - __TBB_ASSERT(my_item_valid(i),"attempt to get invalid item"); - item_type *itm = (tbb::internal::punned_cast(&(item(i).first))); - return *(const item_type *)itm; - } - - // may be called with an empty slot or a slot that has already been constructed into. - void set_my_item(size_t i, const item_type &o) { - if(item(i).second != no_item) { - destroy_item(i); - } - new(&(item(i).first)) item_type(o); - item(i).second = has_item; - } - - // destructively-fetch an object from the buffer - void fetch_item(size_t i, item_type &o) { - __TBB_ASSERT(my_item_valid(i), "Trying to fetch an empty slot"); - o = get_my_item(i); // could have std::move assign semantics - destroy_item(i); - } - - // move an existing item from one slot to another. The moved-to slot must be unoccupied, - // the moved-from slot must exist and not be reserved. The after, from will be empty, - // to will be occupied but not reserved - void move_item(size_t to, size_t from) { - __TBB_ASSERT(!my_item_valid(to), "Trying to move to a non-empty slot"); - __TBB_ASSERT(my_item_valid(from), "Trying to move from an empty slot"); - set_my_item(to, get_my_item(from)); // could have std::move semantics - destroy_item(from); - - } - - // put an item in an empty slot. Return true if successful, else false - bool place_item(size_t here, const item_type &me) { -#if !TBB_DEPRECATED_SEQUENCER_DUPLICATES - if(my_item_valid(here)) return false; -#endif - set_my_item(here, me); - return true; - } - - // could be implemented with std::move semantics - void swap_items(size_t i, size_t j) { - __TBB_ASSERT(my_item_valid(i) && my_item_valid(j), "attempt to swap invalid item(s)"); - item_type temp = get_my_item(i); - set_my_item(i, get_my_item(j)); - set_my_item(j, temp); - } - - void destroy_item(size_type i) { - __TBB_ASSERT(my_item_valid(i), "destruction of invalid item"); - (tbb::internal::punned_cast(&(item(i).first)))->~item_type(); - item(i).second = no_item; - } - - // returns the front element - const item_type& front() const - { - __TBB_ASSERT(my_item_valid(my_head), "attempt to fetch head non-item"); - return get_my_item(my_head); - } - - // returns the back element - const item_type& back() const - { - __TBB_ASSERT(my_item_valid(my_tail - 1), "attempt to fetch head non-item"); - return get_my_item(my_tail - 1); - } - - // following methods are for reservation of the front of a buffer. - void reserve_item(size_type i) { __TBB_ASSERT(my_item_valid(i) && !my_item_reserved(i), "item cannot be reserved"); item(i).second = reserved_item; } - void release_item(size_type i) { __TBB_ASSERT(my_item_reserved(i), "item is not reserved"); item(i).second = has_item; } - - void destroy_front() { destroy_item(my_head); ++my_head; } - void destroy_back() { destroy_item(my_tail-1); --my_tail; } - - // we have to be able to test against a new tail value without changing my_tail - // grow_array doesn't work if we change my_tail when the old array is too small - size_type size(size_t new_tail = 0) { return (new_tail ? new_tail : my_tail) - my_head; } - size_type capacity() { return my_array_size; } - // sequencer_node does not use this method, so we don't - // need a version that passes in the new_tail value. - bool buffer_full() { return size() >= capacity(); } - - //! Grows the internal array. - void grow_my_array( size_t minimum_size ) { - // test that we haven't made the structure inconsistent. - __TBB_ASSERT(capacity() >= my_tail - my_head, "total items exceed capacity"); - size_type new_size = my_array_size ? 2*my_array_size : initial_buffer_size; - while( new_sizeback(); - destroy_back(); - return true; - } - - bool pop_front(item_type &v) { - if(!my_item_valid(my_head)) { - return false; - } - v = this->front(); - destroy_front(); - return true; - } - - // This is used both for reset and for grow_my_array. In the case of grow_my_array - // we want to retain the values of the head and tail. - void clean_up_buffer(bool reset_pointers) { - if (my_array) { - for( size_type i=my_head; i > - class reservable_item_buffer : public item_buffer { - protected: - using item_buffer::my_item_valid; - using item_buffer::my_head; - - public: - reservable_item_buffer() : item_buffer(), my_reserved(false) {} - void reset() {my_reserved = false; item_buffer::reset(); } - protected: - - bool reserve_front(T &v) { - if(my_reserved || !my_item_valid(this->my_head)) return false; - my_reserved = true; - // reserving the head - v = this->front(); - this->reserve_item(this->my_head); - return true; - } - - void consume_front() { - __TBB_ASSERT(my_reserved, "Attempt to consume a non-reserved item"); - this->destroy_front(); - my_reserved = false; - } - - void release_front() { - __TBB_ASSERT(my_reserved, "Attempt to release a non-reserved item"); - this->release_item(this->my_head); - my_reserved = false; - } - - bool my_reserved; - }; - -} // namespace internal - -#endif // __TBB__flow_graph_item_buffer_impl_H diff --git a/src/3rd party/tbb/internal/_flow_graph_join_impl.h b/src/3rd party/tbb/internal/_flow_graph_join_impl.h deleted file mode 100644 index b879b85270..0000000000 --- a/src/3rd party/tbb/internal/_flow_graph_join_impl.h +++ /dev/null @@ -1,2002 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB__flow_graph_join_impl_H -#define __TBB__flow_graph_join_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -namespace internal { - - struct forwarding_base : tbb::internal::no_assign { - forwarding_base(graph &g) : graph_ref(g) {} - virtual ~forwarding_base() {} - // decrement_port_count may create a forwarding task. If we cannot handle the task - // ourselves, ask decrement_port_count to deal with it. - virtual task * decrement_port_count(bool handle_task) = 0; - virtual void increment_port_count() = 0; - // moved here so input ports can queue tasks - graph& graph_ref; - }; - - // specialization that lets us keep a copy of the current_key for building results. - // KeyType can be a reference type. - template - struct matching_forwarding_base : public forwarding_base { - typedef typename tbb::internal::strip::type current_key_type; - matching_forwarding_base(graph &g) : forwarding_base(g) { } - virtual task * increment_key_count(current_key_type const & /*t*/, bool /*handle_task*/) = 0; // {return NULL;} - current_key_type current_key; // so ports can refer to FE's desired items - }; - - template< int N > - struct join_helper { - - template< typename TupleType, typename PortType > - static inline void set_join_node_pointer(TupleType &my_input, PortType *port) { - tbb::flow::get( my_input ).set_join_node_pointer(port); - join_helper::set_join_node_pointer( my_input, port ); - } - template< typename TupleType > - static inline void consume_reservations( TupleType &my_input ) { - tbb::flow::get( my_input ).consume(); - join_helper::consume_reservations( my_input ); - } - - template< typename TupleType > - static inline void release_my_reservation( TupleType &my_input ) { - tbb::flow::get( my_input ).release(); - } - - template - static inline void release_reservations( TupleType &my_input) { - join_helper::release_reservations(my_input); - release_my_reservation(my_input); - } - - template< typename InputTuple, typename OutputTuple > - static inline bool reserve( InputTuple &my_input, OutputTuple &out) { - if ( !tbb::flow::get( my_input ).reserve( tbb::flow::get( out ) ) ) return false; - if ( !join_helper::reserve( my_input, out ) ) { - release_my_reservation( my_input ); - return false; - } - return true; - } - - template - static inline bool get_my_item( InputTuple &my_input, OutputTuple &out) { - bool res = tbb::flow::get(my_input).get_item(tbb::flow::get(out) ); // may fail - return join_helper::get_my_item(my_input, out) && res; // do get on other inputs before returning - } - - template - static inline bool get_items(InputTuple &my_input, OutputTuple &out) { - return get_my_item(my_input, out); - } - - template - static inline void reset_my_port(InputTuple &my_input) { - join_helper::reset_my_port(my_input); - tbb::flow::get(my_input).reset_port(); - } - - template - static inline void reset_ports(InputTuple& my_input) { - reset_my_port(my_input); - } - - template - static inline void set_key_functors(InputTuple &my_input, KeyFuncTuple &my_key_funcs) { - tbb::flow::get(my_input).set_my_key_func(tbb::flow::get(my_key_funcs)); - tbb::flow::get(my_key_funcs) = NULL; - join_helper::set_key_functors(my_input, my_key_funcs); - } - - template< typename KeyFuncTuple> - static inline void copy_key_functors(KeyFuncTuple &my_inputs, KeyFuncTuple &other_inputs) { - if(tbb::flow::get(other_inputs).get_my_key_func()) { - tbb::flow::get(my_inputs).set_my_key_func(tbb::flow::get(other_inputs).get_my_key_func()->clone()); - } - join_helper::copy_key_functors(my_inputs, other_inputs); - } - - template - static inline void reset_inputs(InputTuple &my_input, reset_flags f) { - join_helper::reset_inputs(my_input, f); - tbb::flow::get(my_input).reset_receiver(f); - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - template - static inline void extract_inputs(InputTuple &my_input) { - join_helper::extract_inputs(my_input); - tbb::flow::get(my_input).extract_receiver(); - } -#endif - }; // join_helper - - template< > - struct join_helper<1> { - - template< typename TupleType, typename PortType > - static inline void set_join_node_pointer(TupleType &my_input, PortType *port) { - tbb::flow::get<0>( my_input ).set_join_node_pointer(port); - } - - template< typename TupleType > - static inline void consume_reservations( TupleType &my_input ) { - tbb::flow::get<0>( my_input ).consume(); - } - - template< typename TupleType > - static inline void release_my_reservation( TupleType &my_input ) { - tbb::flow::get<0>( my_input ).release(); - } - - template - static inline void release_reservations( TupleType &my_input) { - release_my_reservation(my_input); - } - - template< typename InputTuple, typename OutputTuple > - static inline bool reserve( InputTuple &my_input, OutputTuple &out) { - return tbb::flow::get<0>( my_input ).reserve( tbb::flow::get<0>( out ) ); - } - - template - static inline bool get_my_item( InputTuple &my_input, OutputTuple &out) { - return tbb::flow::get<0>(my_input).get_item(tbb::flow::get<0>(out)); - } - - template - static inline bool get_items(InputTuple &my_input, OutputTuple &out) { - return get_my_item(my_input, out); - } - - template - static inline void reset_my_port(InputTuple &my_input) { - tbb::flow::get<0>(my_input).reset_port(); - } - - template - static inline void reset_ports(InputTuple& my_input) { - reset_my_port(my_input); - } - - template - static inline void set_key_functors(InputTuple &my_input, KeyFuncTuple &my_key_funcs) { - tbb::flow::get<0>(my_input).set_my_key_func(tbb::flow::get<0>(my_key_funcs)); - tbb::flow::get<0>(my_key_funcs) = NULL; - } - - template< typename KeyFuncTuple> - static inline void copy_key_functors(KeyFuncTuple &my_inputs, KeyFuncTuple &other_inputs) { - if(tbb::flow::get<0>(other_inputs).get_my_key_func()) { - tbb::flow::get<0>(my_inputs).set_my_key_func(tbb::flow::get<0>(other_inputs).get_my_key_func()->clone()); - } - } - template - static inline void reset_inputs(InputTuple &my_input, reset_flags f) { - tbb::flow::get<0>(my_input).reset_receiver(f); - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - template - static inline void extract_inputs(InputTuple &my_input) { - tbb::flow::get<0>(my_input).extract_receiver(); - } -#endif - }; // join_helper<1> - - //! The two-phase join port - template< typename T > - class reserving_port : public receiver { - public: - typedef T input_type; - typedef typename receiver::predecessor_type predecessor_type; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename receiver::predecessor_list_type predecessor_list_type; - typedef typename receiver::built_predecessors_type built_predecessors_type; -#endif - private: - // ----------- Aggregator ------------ - enum op_type { reg_pred, rem_pred, res_item, rel_res, con_res -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy -#endif - }; - typedef reserving_port class_type; - - class reserving_port_operation : public aggregated_operation { - public: - char type; - union { - T *my_arg; - predecessor_type *my_pred; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - size_t cnt_val; - predecessor_list_type *plist; -#endif - }; - reserving_port_operation(const T& e, op_type t) : - type(char(t)), my_arg(const_cast(&e)) {} - reserving_port_operation(const predecessor_type &s, op_type t) : type(char(t)), - my_pred(const_cast(&s)) {} - reserving_port_operation(op_type t) : type(char(t)) {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator my_aggregator; - - void handle_operations(reserving_port_operation* op_list) { - reserving_port_operation *current; - bool no_predecessors; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - case reg_pred: - no_predecessors = my_predecessors.empty(); - my_predecessors.add(*(current->my_pred)); - if ( no_predecessors ) { - (void) my_join->decrement_port_count(true); // may try to forward - } - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case rem_pred: - my_predecessors.remove(*(current->my_pred)); - if(my_predecessors.empty()) my_join->increment_port_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case res_item: - if ( reserved ) { - __TBB_store_with_release(current->status, FAILED); - } - else if ( my_predecessors.try_reserve( *(current->my_arg) ) ) { - reserved = true; - __TBB_store_with_release(current->status, SUCCEEDED); - } else { - if ( my_predecessors.empty() ) { - my_join->increment_port_count(); - } - __TBB_store_with_release(current->status, FAILED); - } - break; - case rel_res: - reserved = false; - my_predecessors.try_release( ); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case con_res: - reserved = false; - my_predecessors.try_consume( ); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - case add_blt_pred: - my_predecessors.internal_add_built_predecessor(*(current->my_pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case del_blt_pred: - my_predecessors.internal_delete_built_predecessor(*(current->my_pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cnt: - current->cnt_val = my_predecessors.predecessor_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cpy: - my_predecessors.copy_predecessors(*(current->plist)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - } - } - } - - protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task *try_put_task( const T & ) __TBB_override { - return NULL; - } - - graph& graph_reference() const __TBB_override { - return my_join->graph_ref; - } - - public: - - //! Constructor - reserving_port() : reserved(false) { - my_join = NULL; - my_predecessors.set_owner( this ); - my_aggregator.initialize_handler(handler_type(this)); - } - - // copy constructor - reserving_port(const reserving_port& /* other */) : receiver() { - reserved = false; - my_join = NULL; - my_predecessors.set_owner( this ); - my_aggregator.initialize_handler(handler_type(this)); - } - - void set_join_node_pointer(forwarding_base *join) { - my_join = join; - } - - //! Add a predecessor - bool register_predecessor( predecessor_type &src ) __TBB_override { - reserving_port_operation op_data(src, reg_pred); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - //! Remove a predecessor - bool remove_predecessor( predecessor_type &src ) __TBB_override { - reserving_port_operation op_data(src, rem_pred); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - //! Reserve an item from the port - bool reserve( T &v ) { - reserving_port_operation op_data(v, res_item); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - //! Release the port - void release( ) { - reserving_port_operation op_data(rel_res); - my_aggregator.execute(&op_data); - } - - //! Complete use of the port - void consume( ) { - reserving_port_operation op_data(con_res); - my_aggregator.execute(&op_data); - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - built_predecessors_type &built_predecessors() __TBB_override { return my_predecessors.built_predecessors(); } - void internal_add_built_predecessor(predecessor_type &src) __TBB_override { - reserving_port_operation op_data(src, add_blt_pred); - my_aggregator.execute(&op_data); - } - - void internal_delete_built_predecessor(predecessor_type &src) __TBB_override { - reserving_port_operation op_data(src, del_blt_pred); - my_aggregator.execute(&op_data); - } - - size_t predecessor_count() __TBB_override { - reserving_port_operation op_data(blt_pred_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_predecessors(predecessor_list_type &l) __TBB_override { - reserving_port_operation op_data(blt_pred_cpy); - op_data.plist = &l; - my_aggregator.execute(&op_data); - } - - void extract_receiver() { - my_predecessors.built_predecessors().receiver_extract(*this); - } - -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - - void reset_receiver( reset_flags f) __TBB_override { - if(f & rf_clear_edges) my_predecessors.clear(); - else - my_predecessors.reset(); - reserved = false; - __TBB_ASSERT(!(f&rf_clear_edges) || my_predecessors.empty(), "port edges not removed"); - } - - private: -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - friend class get_graph_helper; -#endif - - forwarding_base *my_join; - reservable_predecessor_cache< T, null_mutex > my_predecessors; - bool reserved; - }; // reserving_port - - //! queueing join_port - template - class queueing_port : public receiver, public item_buffer { - public: - typedef T input_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef queueing_port class_type; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename receiver::built_predecessors_type built_predecessors_type; - typedef typename receiver::predecessor_list_type predecessor_list_type; -#endif - - // ----------- Aggregator ------------ - private: - enum op_type { get__item, res_port, try__put_task -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy -#endif - }; - - class queueing_port_operation : public aggregated_operation { - public: - char type; - T my_val; - T *my_arg; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - predecessor_type *pred; - size_t cnt_val; - predecessor_list_type *plist; -#endif - task * bypass_t; - // constructor for value parameter - queueing_port_operation(const T& e, op_type t) : - type(char(t)), my_val(e) - , bypass_t(NULL) - {} - // constructor for pointer parameter - queueing_port_operation(const T* p, op_type t) : - type(char(t)), my_arg(const_cast(p)) - , bypass_t(NULL) - {} - // constructor with no parameter - queueing_port_operation(op_type t) : type(char(t)) - , bypass_t(NULL) - {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator my_aggregator; - - void handle_operations(queueing_port_operation* op_list) { - queueing_port_operation *current; - bool was_empty; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - case try__put_task: { - task *rtask = NULL; - was_empty = this->buffer_empty(); - this->push_back(current->my_val); - if (was_empty) rtask = my_join->decrement_port_count(false); - else - rtask = SUCCESSFULLY_ENQUEUED; - current->bypass_t = rtask; - __TBB_store_with_release(current->status, SUCCEEDED); - } - break; - case get__item: - if(!this->buffer_empty()) { - *(current->my_arg) = this->front(); - __TBB_store_with_release(current->status, SUCCEEDED); - } - else { - __TBB_store_with_release(current->status, FAILED); - } - break; - case res_port: - __TBB_ASSERT(this->my_item_valid(this->my_head), "No item to reset"); - this->destroy_front(); - if(this->my_item_valid(this->my_head)) { - (void)my_join->decrement_port_count(true); - } - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - case add_blt_pred: - my_built_predecessors.add_edge(*(current->pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case del_blt_pred: - my_built_predecessors.delete_edge(*(current->pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cnt: - current->cnt_val = my_built_predecessors.edge_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cpy: - my_built_predecessors.copy_edges(*(current->plist)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - } - } - } - // ------------ End Aggregator --------------- - - protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task *try_put_task(const T &v) __TBB_override { - queueing_port_operation op_data(v, try__put_task); - my_aggregator.execute(&op_data); - __TBB_ASSERT(op_data.status == SUCCEEDED || !op_data.bypass_t, "inconsistent return from aggregator"); - if(!op_data.bypass_t) return SUCCESSFULLY_ENQUEUED; - return op_data.bypass_t; - } - - graph& graph_reference() const __TBB_override { - return my_join->graph_ref; - } - - public: - - //! Constructor - queueing_port() : item_buffer() { - my_join = NULL; - my_aggregator.initialize_handler(handler_type(this)); - } - - //! copy constructor - queueing_port(const queueing_port& /* other */) : receiver(), item_buffer() { - my_join = NULL; - my_aggregator.initialize_handler(handler_type(this)); - } - - //! record parent for tallying available items - void set_join_node_pointer(forwarding_base *join) { - my_join = join; - } - - bool get_item( T &v ) { - queueing_port_operation op_data(&v, get__item); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - // reset_port is called when item is accepted by successor, but - // is initiated by join_node. - void reset_port() { - queueing_port_operation op_data(res_port); - my_aggregator.execute(&op_data); - return; - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - void internal_add_built_predecessor(predecessor_type &p) __TBB_override { - queueing_port_operation op_data(add_blt_pred); - op_data.pred = &p; - my_aggregator.execute(&op_data); - } - - void internal_delete_built_predecessor(predecessor_type &p) __TBB_override { - queueing_port_operation op_data(del_blt_pred); - op_data.pred = &p; - my_aggregator.execute(&op_data); - } - - size_t predecessor_count() __TBB_override { - queueing_port_operation op_data(blt_pred_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_predecessors(predecessor_list_type &l) __TBB_override { - queueing_port_operation op_data(blt_pred_cpy); - op_data.plist = &l; - my_aggregator.execute(&op_data); - } - - void extract_receiver() { - item_buffer::reset(); - my_built_predecessors.receiver_extract(*this); - } -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - - void reset_receiver(reset_flags f) __TBB_override { - tbb::internal::suppress_unused_warning(f); - item_buffer::reset(); -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - if (f & rf_clear_edges) - my_built_predecessors.clear(); -#endif - } - - private: -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - friend class get_graph_helper; -#endif - - forwarding_base *my_join; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - edge_container my_built_predecessors; -#endif - }; // queueing_port - -#include "_flow_graph_tagged_buffer_impl.h" - - template - struct count_element { - K my_key; - size_t my_value; - }; - - // method to access the key in the counting table - // the ref has already been removed from K - template< typename K > - struct key_to_count_functor { - typedef count_element table_item_type; - const K& operator()(const table_item_type& v) { return v.my_key; } - }; - - // the ports can have only one template parameter. We wrap the types needed in - // a traits type - template< class TraitsType > - class key_matching_port : - public receiver, - public hash_buffer< typename TraitsType::K, typename TraitsType::T, typename TraitsType::TtoK, - typename TraitsType::KHash > { - public: - typedef TraitsType traits; - typedef key_matching_port class_type; - typedef typename TraitsType::T input_type; - typedef typename TraitsType::K key_type; - typedef typename tbb::internal::strip::type noref_key_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef typename TraitsType::TtoK type_to_key_func_type; - typedef typename TraitsType::KHash hash_compare_type; - typedef hash_buffer< key_type, input_type, type_to_key_func_type, hash_compare_type > buffer_type; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename receiver::built_predecessors_type built_predecessors_type; - typedef typename receiver::predecessor_list_type predecessor_list_type; -#endif - private: -// ----------- Aggregator ------------ - private: - enum op_type { try__put, get__item, res_port -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy -#endif - }; - - class key_matching_port_operation : public aggregated_operation { - public: - char type; - input_type my_val; - input_type *my_arg; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - predecessor_type *pred; - size_t cnt_val; - predecessor_list_type *plist; -#endif - // constructor for value parameter - key_matching_port_operation(const input_type& e, op_type t) : - type(char(t)), my_val(e) {} - // constructor for pointer parameter - key_matching_port_operation(const input_type* p, op_type t) : - type(char(t)), my_arg(const_cast(p)) {} - // constructor with no parameter - key_matching_port_operation(op_type t) : type(char(t)) {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator my_aggregator; - - void handle_operations(key_matching_port_operation* op_list) { - key_matching_port_operation *current; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - case try__put: { - bool was_inserted = this->insert_with_key(current->my_val); - // return failure if a duplicate insertion occurs - __TBB_store_with_release(current->status, was_inserted ? SUCCEEDED : FAILED); - } - break; - case get__item: - // use current_key from FE for item - if(!this->find_with_key(my_join->current_key, *(current->my_arg))) { - __TBB_ASSERT(false, "Failed to find item corresponding to current_key."); - } - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case res_port: - // use current_key from FE for item - this->delete_with_key(my_join->current_key); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - case add_blt_pred: - my_built_predecessors.add_edge(*(current->pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case del_blt_pred: - my_built_predecessors.delete_edge(*(current->pred)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cnt: - current->cnt_val = my_built_predecessors.edge_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_pred_cpy: - my_built_predecessors.copy_edges(*(current->plist)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#endif - } - } - } -// ------------ End Aggregator --------------- - protected: - template< typename R, typename B > friend class run_and_put_task; - template friend class internal::broadcast_cache; - template friend class internal::round_robin_cache; - task *try_put_task(const input_type& v) __TBB_override { - key_matching_port_operation op_data(v, try__put); - task *rtask = NULL; - my_aggregator.execute(&op_data); - if(op_data.status == SUCCEEDED) { - rtask = my_join->increment_key_count((*(this->get_key_func()))(v), false); // may spawn - // rtask has to reflect the return status of the try_put - if(!rtask) rtask = SUCCESSFULLY_ENQUEUED; - } - return rtask; - } - - graph& graph_reference() const __TBB_override { - return my_join->graph_ref; - } - - public: - - key_matching_port() : receiver(), buffer_type() { - my_join = NULL; - my_aggregator.initialize_handler(handler_type(this)); - } - - // copy constructor - key_matching_port(const key_matching_port& /*other*/) : receiver(), buffer_type() { - my_join = NULL; - my_aggregator.initialize_handler(handler_type(this)); - } - - ~key_matching_port() { } - - void set_join_node_pointer(forwarding_base *join) { - my_join = dynamic_cast*>(join); - } - - void set_my_key_func(type_to_key_func_type *f) { this->set_key_func(f); } - - type_to_key_func_type* get_my_key_func() { return this->get_key_func(); } - - bool get_item( input_type &v ) { - // aggregator uses current_key from FE for Key - key_matching_port_operation op_data(&v, get__item); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } - - void internal_add_built_predecessor(predecessor_type &p) __TBB_override { - key_matching_port_operation op_data(add_blt_pred); - op_data.pred = &p; - my_aggregator.execute(&op_data); - } - - void internal_delete_built_predecessor(predecessor_type &p) __TBB_override { - key_matching_port_operation op_data(del_blt_pred); - op_data.pred = &p; - my_aggregator.execute(&op_data); - } - - size_t predecessor_count() __TBB_override { - key_matching_port_operation op_data(blt_pred_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_predecessors(predecessor_list_type &l) __TBB_override { - key_matching_port_operation op_data(blt_pred_cpy); - op_data.plist = &l; - my_aggregator.execute(&op_data); - } -#endif - - // reset_port is called when item is accepted by successor, but - // is initiated by join_node. - void reset_port() { - key_matching_port_operation op_data(res_port); - my_aggregator.execute(&op_data); - return; - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - void extract_receiver() { - buffer_type::reset(); - my_built_predecessors.receiver_extract(*this); - } -#endif - void reset_receiver(reset_flags f ) __TBB_override { - tbb::internal::suppress_unused_warning(f); - buffer_type::reset(); -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - if (f & rf_clear_edges) - my_built_predecessors.clear(); -#endif - } - - private: - // my_join forwarding base used to count number of inputs that - // received key. - matching_forwarding_base *my_join; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - edge_container my_built_predecessors; -#endif - }; // key_matching_port - - using namespace graph_policy_namespace; - - template - class join_node_base; - - //! join_node_FE : implements input port policy - template - class join_node_FE; - - template - class join_node_FE : public forwarding_base { - public: - static const int N = tbb::flow::tuple_size::value; - typedef OutputTuple output_type; - typedef InputTuple input_type; - typedef join_node_base base_node_type; // for forwarding - - join_node_FE(graph &g) : forwarding_base(g), my_node(NULL) { - ports_with_no_inputs = N; - join_helper::set_join_node_pointer(my_inputs, this); - } - - join_node_FE(const join_node_FE& other) : forwarding_base((other.forwarding_base::graph_ref)), my_node(NULL) { - ports_with_no_inputs = N; - join_helper::set_join_node_pointer(my_inputs, this); - } - - void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; } - - void increment_port_count() __TBB_override { - ++ports_with_no_inputs; - } - - // if all input_ports have predecessors, spawn forward to try and consume tuples - task * decrement_port_count(bool handle_task) __TBB_override { - if(ports_with_no_inputs.fetch_and_decrement() == 1) { - if(internal::is_graph_active(this->graph_ref)) { - task *rtask = new ( task::allocate_additional_child_of( *(this->graph_ref.root_task()) ) ) - forward_task_bypass(*my_node); - if(!handle_task) return rtask; - internal::spawn_in_graph_arena(this->graph_ref, *rtask); - } - } - return NULL; - } - - input_type &input_ports() { return my_inputs; } - - protected: - - void reset( reset_flags f) { - // called outside of parallel contexts - ports_with_no_inputs = N; - join_helper::reset_inputs(my_inputs, f); - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - void extract( ) { - // called outside of parallel contexts - ports_with_no_inputs = N; - join_helper::extract_inputs(my_inputs); - } -#endif - - // all methods on input ports should be called under mutual exclusion from join_node_base. - - bool tuple_build_may_succeed() { - return !ports_with_no_inputs; - } - - bool try_to_make_tuple(output_type &out) { - if(ports_with_no_inputs) return false; - return join_helper::reserve(my_inputs, out); - } - - void tuple_accepted() { - join_helper::consume_reservations(my_inputs); - } - void tuple_rejected() { - join_helper::release_reservations(my_inputs); - } - - input_type my_inputs; - base_node_type *my_node; - atomic ports_with_no_inputs; - }; // join_node_FE - - template - class join_node_FE : public forwarding_base { - public: - static const int N = tbb::flow::tuple_size::value; - typedef OutputTuple output_type; - typedef InputTuple input_type; - typedef join_node_base base_node_type; // for forwarding - - join_node_FE(graph &g) : forwarding_base(g), my_node(NULL) { - ports_with_no_items = N; - join_helper::set_join_node_pointer(my_inputs, this); - } - - join_node_FE(const join_node_FE& other) : forwarding_base((other.forwarding_base::graph_ref)), my_node(NULL) { - ports_with_no_items = N; - join_helper::set_join_node_pointer(my_inputs, this); - } - - // needed for forwarding - void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; } - - void reset_port_count() { - ports_with_no_items = N; - } - - // if all input_ports have items, spawn forward to try and consume tuples - task * decrement_port_count(bool handle_task) __TBB_override - { - if(ports_with_no_items.fetch_and_decrement() == 1) { - if(internal::is_graph_active(this->graph_ref)) { - task *rtask = new ( task::allocate_additional_child_of( *(this->graph_ref.root_task()) ) ) - forward_task_bypass (*my_node); - if(!handle_task) return rtask; - internal::spawn_in_graph_arena(this->graph_ref, *rtask); - } - } - return NULL; - } - - void increment_port_count() __TBB_override { __TBB_ASSERT(false, NULL); } // should never be called - - input_type &input_ports() { return my_inputs; } - - protected: - - void reset( reset_flags f) { - reset_port_count(); - join_helper::reset_inputs(my_inputs, f ); - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - void extract() { - reset_port_count(); - join_helper::extract_inputs(my_inputs); - } -#endif - // all methods on input ports should be called under mutual exclusion from join_node_base. - - bool tuple_build_may_succeed() { - return !ports_with_no_items; - } - - bool try_to_make_tuple(output_type &out) { - if(ports_with_no_items) return false; - return join_helper::get_items(my_inputs, out); - } - - void tuple_accepted() { - reset_port_count(); - join_helper::reset_ports(my_inputs); - } - void tuple_rejected() { - // nothing to do. - } - - input_type my_inputs; - base_node_type *my_node; - atomic ports_with_no_items; - }; // join_node_FE - - // key_matching join front-end. - template - class join_node_FE, InputTuple, OutputTuple> : public matching_forwarding_base, - // buffer of key value counts - public hash_buffer< // typedefed below to key_to_count_buffer_type - typename tbb::internal::strip::type&, // force ref type on K - count_element::type>, - internal::type_to_key_function_body< - count_element::type>, - typename tbb::internal::strip::type& >, - KHash >, - // buffer of output items - public item_buffer { - public: - static const int N = tbb::flow::tuple_size::value; - typedef OutputTuple output_type; - typedef InputTuple input_type; - typedef K key_type; - typedef typename tbb::internal::strip::type unref_key_type; - typedef KHash key_hash_compare; - // must use K without ref. - typedef count_element count_element_type; - // method that lets us refer to the key of this type. - typedef key_to_count_functor key_to_count_func; - typedef internal::type_to_key_function_body< count_element_type, unref_key_type&> TtoK_function_body_type; - typedef internal::type_to_key_function_body_leaf TtoK_function_body_leaf_type; - // this is the type of the special table that keeps track of the number of discrete - // elements corresponding to each key that we've seen. - typedef hash_buffer< unref_key_type&, count_element_type, TtoK_function_body_type, key_hash_compare > - key_to_count_buffer_type; - typedef item_buffer output_buffer_type; - typedef join_node_base, InputTuple, OutputTuple> base_node_type; // for forwarding - typedef matching_forwarding_base forwarding_base_type; - -// ----------- Aggregator ------------ - // the aggregator is only needed to serialize the access to the hash table. - // and the output_buffer_type base class - private: - enum op_type { res_count, inc_count, may_succeed, try_make }; - typedef join_node_FE, InputTuple, OutputTuple> class_type; - - class key_matching_FE_operation : public aggregated_operation { - public: - char type; - unref_key_type my_val; - output_type* my_output; - task *bypass_t; - bool enqueue_task; - // constructor for value parameter - key_matching_FE_operation(const unref_key_type& e , bool q_task , op_type t) : type(char(t)), my_val(e), - my_output(NULL), bypass_t(NULL), enqueue_task(q_task) {} - key_matching_FE_operation(output_type *p, op_type t) : type(char(t)), my_output(p), bypass_t(NULL), - enqueue_task(true) {} - // constructor with no parameter - key_matching_FE_operation(op_type t) : type(char(t)), my_output(NULL), bypass_t(NULL), enqueue_task(true) {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator my_aggregator; - - // called from aggregator, so serialized - // returns a task pointer if the a task would have been enqueued but we asked that - // it be returned. Otherwise returns NULL. - task * fill_output_buffer(unref_key_type &t, bool should_enqueue, bool handle_task) { - output_type l_out; - task *rtask = NULL; - bool do_fwd = should_enqueue && this->buffer_empty() && internal::is_graph_active(this->graph_ref); - this->current_key = t; - this->delete_with_key(this->current_key); // remove the key - if(join_helper::get_items(my_inputs, l_out)) { // <== call back - this->push_back(l_out); - if(do_fwd) { // we enqueue if receiving an item from predecessor, not if successor asks for item - rtask = new ( task::allocate_additional_child_of( *(this->graph_ref.root_task()) ) ) - forward_task_bypass(*my_node); - if(handle_task) { - internal::spawn_in_graph_arena(this->graph_ref, *rtask); - rtask = NULL; - } - do_fwd = false; - } - // retire the input values - join_helper::reset_ports(my_inputs); // <== call back - } - else { - __TBB_ASSERT(false, "should have had something to push"); - } - return rtask; - } - - void handle_operations(key_matching_FE_operation* op_list) { - key_matching_FE_operation *current; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - case res_count: // called from BE - { - this->destroy_front(); - __TBB_store_with_release(current->status, SUCCEEDED); - } - break; - case inc_count: { // called from input ports - count_element_type *p = 0; - unref_key_type &t = current->my_val; - bool do_enqueue = current->enqueue_task; - if(!(this->find_ref_with_key(t,p))) { - count_element_type ev; - ev.my_key = t; - ev.my_value = 0; - this->insert_with_key(ev); - if(!(this->find_ref_with_key(t,p))) { - __TBB_ASSERT(false, "should find key after inserting it"); - } - } - if(++(p->my_value) == size_t(N)) { - task *rtask = fill_output_buffer(t, true, do_enqueue); - __TBB_ASSERT(!rtask || !do_enqueue, "task should not be returned"); - current->bypass_t = rtask; - } - } - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case may_succeed: // called from BE - __TBB_store_with_release(current->status, this->buffer_empty() ? FAILED : SUCCEEDED); - break; - case try_make: // called from BE - if(this->buffer_empty()) { - __TBB_store_with_release(current->status, FAILED); - } - else { - *(current->my_output) = this->front(); - __TBB_store_with_release(current->status, SUCCEEDED); - } - break; - } - } - } -// ------------ End Aggregator --------------- - - public: - template - join_node_FE(graph &g, FunctionTuple &TtoK_funcs) : forwarding_base_type(g), my_node(NULL) { - join_helper::set_join_node_pointer(my_inputs, this); - join_helper::set_key_functors(my_inputs, TtoK_funcs); - my_aggregator.initialize_handler(handler_type(this)); - TtoK_function_body_type *cfb = new TtoK_function_body_leaf_type(key_to_count_func()); - this->set_key_func(cfb); - } - - join_node_FE(const join_node_FE& other) : forwarding_base_type((other.forwarding_base_type::graph_ref)), key_to_count_buffer_type(), - output_buffer_type() { - my_node = NULL; - join_helper::set_join_node_pointer(my_inputs, this); - join_helper::copy_key_functors(my_inputs, const_cast(other.my_inputs)); - my_aggregator.initialize_handler(handler_type(this)); - TtoK_function_body_type *cfb = new TtoK_function_body_leaf_type(key_to_count_func()); - this->set_key_func(cfb); - } - - // needed for forwarding - void set_my_node(base_node_type *new_my_node) { my_node = new_my_node; } - - void reset_port_count() { // called from BE - key_matching_FE_operation op_data(res_count); - my_aggregator.execute(&op_data); - return; - } - - // if all input_ports have items, spawn forward to try and consume tuples - // return a task if we are asked and did create one. - task *increment_key_count(unref_key_type const & t, bool handle_task) __TBB_override { // called from input_ports - key_matching_FE_operation op_data(t, handle_task, inc_count); - my_aggregator.execute(&op_data); - return op_data.bypass_t; - } - - task *decrement_port_count(bool /*handle_task*/) __TBB_override { __TBB_ASSERT(false, NULL); return NULL; } - - void increment_port_count() __TBB_override { __TBB_ASSERT(false, NULL); } // should never be called - - input_type &input_ports() { return my_inputs; } - - protected: - - void reset( reset_flags f ) { - // called outside of parallel contexts - join_helper::reset_inputs(my_inputs, f); - - key_to_count_buffer_type::reset(); - output_buffer_type::reset(); - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - void extract() { - // called outside of parallel contexts - join_helper::extract_inputs(my_inputs); - key_to_count_buffer_type::reset(); // have to reset the tag counts - output_buffer_type::reset(); // also the queue of outputs - // my_node->current_tag = NO_TAG; - } -#endif - // all methods on input ports should be called under mutual exclusion from join_node_base. - - bool tuple_build_may_succeed() { // called from back-end - key_matching_FE_operation op_data(may_succeed); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - // cannot lock while calling back to input_ports. current_key will only be set - // and reset under the aggregator, so it will remain consistent. - bool try_to_make_tuple(output_type &out) { - key_matching_FE_operation op_data(&out,try_make); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - void tuple_accepted() { - reset_port_count(); // reset current_key after ports reset. - } - - void tuple_rejected() { - // nothing to do. - } - - input_type my_inputs; // input ports - base_node_type *my_node; - }; // join_node_FE, InputTuple, OutputTuple> - - //! join_node_base - template - class join_node_base : public graph_node, public join_node_FE, - public sender { - protected: - using graph_node::my_graph; - public: - typedef OutputTuple output_type; - - typedef typename sender::successor_type successor_type; - typedef join_node_FE input_ports_type; - using input_ports_type::tuple_build_may_succeed; - using input_ports_type::try_to_make_tuple; - using input_ports_type::tuple_accepted; - using input_ports_type::tuple_rejected; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename sender::built_successors_type built_successors_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - - private: - // ----------- Aggregator ------------ - enum op_type { reg_succ, rem_succ, try__get, do_fwrd, do_fwrd_bypass -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - , add_blt_succ, del_blt_succ, blt_succ_cnt, blt_succ_cpy -#endif - }; - typedef join_node_base class_type; - - class join_node_base_operation : public aggregated_operation { - public: - char type; - union { - output_type *my_arg; - successor_type *my_succ; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - size_t cnt_val; - successor_list_type *slist; -#endif - }; - task *bypass_t; - join_node_base_operation(const output_type& e, op_type t) : type(char(t)), - my_arg(const_cast(&e)), bypass_t(NULL) {} - join_node_base_operation(const successor_type &s, op_type t) : type(char(t)), - my_succ(const_cast(&s)), bypass_t(NULL) {} - join_node_base_operation(op_type t) : type(char(t)), bypass_t(NULL) {} - }; - - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - bool forwarder_busy; - aggregator my_aggregator; - - void handle_operations(join_node_base_operation* op_list) { - join_node_base_operation *current; - while(op_list) { - current = op_list; - op_list = op_list->next; - switch(current->type) { - case reg_succ: { - my_successors.register_successor(*(current->my_succ)); - if(tuple_build_may_succeed() && !forwarder_busy && internal::is_graph_active(my_graph)) { - task *rtask = new ( task::allocate_additional_child_of(*(my_graph.root_task())) ) - forward_task_bypass - >(*this); - internal::spawn_in_graph_arena(my_graph, *rtask); - forwarder_busy = true; - } - __TBB_store_with_release(current->status, SUCCEEDED); - } - break; - case rem_succ: - my_successors.remove_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case try__get: - if(tuple_build_may_succeed()) { - if(try_to_make_tuple(*(current->my_arg))) { - tuple_accepted(); - __TBB_store_with_release(current->status, SUCCEEDED); - } - else __TBB_store_with_release(current->status, FAILED); - } - else __TBB_store_with_release(current->status, FAILED); - break; - case do_fwrd_bypass: { - bool build_succeeded; - task *last_task = NULL; - output_type out; - if(tuple_build_may_succeed()) { // checks output queue of FE - do { - build_succeeded = try_to_make_tuple(out); // fetch front_end of queue - if(build_succeeded) { - task *new_task = my_successors.try_put_task(out); - last_task = combine_tasks(my_graph, last_task, new_task); - if(new_task) { - tuple_accepted(); - } - else { - tuple_rejected(); - build_succeeded = false; - } - } - } while(build_succeeded); - } - current->bypass_t = last_task; - __TBB_store_with_release(current->status, SUCCEEDED); - forwarder_busy = false; - } - break; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - case add_blt_succ: - my_successors.internal_add_built_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case del_blt_succ: - my_successors.internal_delete_built_successor(*(current->my_succ)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_succ_cnt: - current->cnt_val = my_successors.successor_count(); - __TBB_store_with_release(current->status, SUCCEEDED); - break; - case blt_succ_cpy: - my_successors.copy_successors(*(current->slist)); - __TBB_store_with_release(current->status, SUCCEEDED); - break; -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - } - } - } - // ---------- end aggregator ----------- - public: - join_node_base(graph &g) : graph_node(g), input_ports_type(g), forwarder_busy(false) { - my_successors.set_owner(this); - input_ports_type::set_my_node(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - join_node_base(const join_node_base& other) : - graph_node(other.graph_node::my_graph), input_ports_type(other), - sender(), forwarder_busy(false), my_successors() { - my_successors.set_owner(this); - input_ports_type::set_my_node(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - template - join_node_base(graph &g, FunctionTuple f) : graph_node(g), input_ports_type(g, f), forwarder_busy(false) { - my_successors.set_owner(this); - input_ports_type::set_my_node(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - bool register_successor(successor_type &r) __TBB_override { - join_node_base_operation op_data(r, reg_succ); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - bool remove_successor( successor_type &r) __TBB_override { - join_node_base_operation op_data(r, rem_succ); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - - bool try_get( output_type &v) __TBB_override { - join_node_base_operation op_data(v, try__get); - my_aggregator.execute(&op_data); - return op_data.status == SUCCEEDED; - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - built_successors_type &built_successors() __TBB_override { return my_successors.built_successors(); } - - void internal_add_built_successor( successor_type &r) __TBB_override { - join_node_base_operation op_data(r, add_blt_succ); - my_aggregator.execute(&op_data); - } - - void internal_delete_built_successor( successor_type &r) __TBB_override { - join_node_base_operation op_data(r, del_blt_succ); - my_aggregator.execute(&op_data); - } - - size_t successor_count() __TBB_override { - join_node_base_operation op_data(blt_succ_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_successors(successor_list_type &l) __TBB_override { - join_node_base_operation op_data(blt_succ_cpy); - op_data.slist = &l; - my_aggregator.execute(&op_data); - } -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - void extract() __TBB_override { - input_ports_type::extract(); - my_successors.built_successors().sender_extract(*this); - } -#endif - - protected: - - void reset_node(reset_flags f) __TBB_override { - input_ports_type::reset(f); - if(f & rf_clear_edges) my_successors.clear(); - } - - private: - broadcast_cache my_successors; - - friend class forward_task_bypass< join_node_base >; - task *forward_task() { - join_node_base_operation op_data(do_fwrd_bypass); - my_aggregator.execute(&op_data); - return op_data.bypass_t; - } - - }; // join_node_base - - // join base class type generator - template class PT, typename OutputTuple, typename JP> - struct join_base { - typedef typename internal::join_node_base::type, OutputTuple> type; - }; - - template - struct join_base > { - typedef key_matching key_traits_type; - typedef K key_type; - typedef KHash key_hash_compare; - typedef typename internal::join_node_base< key_traits_type, - // ports type - typename wrap_key_tuple_elements::type, - OutputTuple > type; - }; - - //! unfolded_join_node : passes input_ports_type to join_node_base. We build the input port type - // using tuple_element. The class PT is the port type (reserving_port, queueing_port, key_matching_port) - // and should match the typename. - - template class PT, typename OutputTuple, typename JP> - class unfolded_join_node : public join_base::type { - public: - typedef typename wrap_tuple_elements::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base base_type; - public: - unfolded_join_node(graph &g) : base_type(g) {} - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; - -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - template - struct key_from_message_body { - K operator()(const T& t) const { - using tbb::flow::key_from_message; - return key_from_message(t); - } - }; - // Adds const to reference type - template - struct key_from_message_body { - const K& operator()(const T& t) const { - using tbb::flow::key_from_message; - return key_from_message(t); - } - }; -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - // key_matching unfolded_join_node. This must be a separate specialization because the constructors - // differ. - - template - class unfolded_join_node<2,key_matching_port,OutputTuple,key_matching > : public - join_base<2,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - public: - typedef typename wrap_key_tuple_elements<2,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base, input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename tbb::flow::tuple< f0_p, f1_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 2, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; - - template - class unfolded_join_node<3,key_matching_port,OutputTuple,key_matching > : public - join_base<3,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - public: - typedef typename wrap_key_tuple_elements<3,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base, input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 3, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; - - template - class unfolded_join_node<4,key_matching_port,OutputTuple,key_matching > : public - join_base<4,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - public: - typedef typename wrap_key_tuple_elements<4,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base, input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 4, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; - - template - class unfolded_join_node<5,key_matching_port,OutputTuple,key_matching > : public - join_base<5,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - public: - typedef typename wrap_key_tuple_elements<5,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 5, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; - -#if __TBB_VARIADIC_MAX >= 6 - template - class unfolded_join_node<6,key_matching_port,OutputTuple,key_matching > : public - join_base<6,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - typedef typename tbb::flow::tuple_element<5, OutputTuple>::type T5; - public: - typedef typename wrap_key_tuple_elements<6,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename internal::type_to_key_function_body *f5_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, Body5 body5) - : base_type(g, func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4), - new internal::type_to_key_function_body_leaf(body5) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 6, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; -#endif - -#if __TBB_VARIADIC_MAX >= 7 - template - class unfolded_join_node<7,key_matching_port,OutputTuple,key_matching > : public - join_base<7,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - typedef typename tbb::flow::tuple_element<5, OutputTuple>::type T5; - typedef typename tbb::flow::tuple_element<6, OutputTuple>::type T6; - public: - typedef typename wrap_key_tuple_elements<7,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename internal::type_to_key_function_body *f5_p; - typedef typename internal::type_to_key_function_body *f6_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, - Body5 body5, Body6 body6) : base_type(g, func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4), - new internal::type_to_key_function_body_leaf(body5), - new internal::type_to_key_function_body_leaf(body6) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 7, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; -#endif - -#if __TBB_VARIADIC_MAX >= 8 - template - class unfolded_join_node<8,key_matching_port,OutputTuple,key_matching > : public - join_base<8,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - typedef typename tbb::flow::tuple_element<5, OutputTuple>::type T5; - typedef typename tbb::flow::tuple_element<6, OutputTuple>::type T6; - typedef typename tbb::flow::tuple_element<7, OutputTuple>::type T7; - public: - typedef typename wrap_key_tuple_elements<8,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename internal::type_to_key_function_body *f5_p; - typedef typename internal::type_to_key_function_body *f6_p; - typedef typename internal::type_to_key_function_body *f7_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p, f7_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, - Body5 body5, Body6 body6, Body7 body7) : base_type(g, func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4), - new internal::type_to_key_function_body_leaf(body5), - new internal::type_to_key_function_body_leaf(body6), - new internal::type_to_key_function_body_leaf(body7) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 8, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; -#endif - -#if __TBB_VARIADIC_MAX >= 9 - template - class unfolded_join_node<9,key_matching_port,OutputTuple,key_matching > : public - join_base<9,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - typedef typename tbb::flow::tuple_element<5, OutputTuple>::type T5; - typedef typename tbb::flow::tuple_element<6, OutputTuple>::type T6; - typedef typename tbb::flow::tuple_element<7, OutputTuple>::type T7; - typedef typename tbb::flow::tuple_element<8, OutputTuple>::type T8; - public: - typedef typename wrap_key_tuple_elements<9,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename internal::type_to_key_function_body *f5_p; - typedef typename internal::type_to_key_function_body *f6_p; - typedef typename internal::type_to_key_function_body *f7_p; - typedef typename internal::type_to_key_function_body *f8_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p, f7_p, f8_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, - Body5 body5, Body6 body6, Body7 body7, Body8 body8) : base_type(g, func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4), - new internal::type_to_key_function_body_leaf(body5), - new internal::type_to_key_function_body_leaf(body6), - new internal::type_to_key_function_body_leaf(body7), - new internal::type_to_key_function_body_leaf(body8) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 9, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; -#endif - -#if __TBB_VARIADIC_MAX >= 10 - template - class unfolded_join_node<10,key_matching_port,OutputTuple,key_matching > : public - join_base<10,key_matching_port,OutputTuple,key_matching >::type { - typedef typename tbb::flow::tuple_element<0, OutputTuple>::type T0; - typedef typename tbb::flow::tuple_element<1, OutputTuple>::type T1; - typedef typename tbb::flow::tuple_element<2, OutputTuple>::type T2; - typedef typename tbb::flow::tuple_element<3, OutputTuple>::type T3; - typedef typename tbb::flow::tuple_element<4, OutputTuple>::type T4; - typedef typename tbb::flow::tuple_element<5, OutputTuple>::type T5; - typedef typename tbb::flow::tuple_element<6, OutputTuple>::type T6; - typedef typename tbb::flow::tuple_element<7, OutputTuple>::type T7; - typedef typename tbb::flow::tuple_element<8, OutputTuple>::type T8; - typedef typename tbb::flow::tuple_element<9, OutputTuple>::type T9; - public: - typedef typename wrap_key_tuple_elements<10,key_matching_port,key_matching,OutputTuple>::type input_ports_type; - typedef OutputTuple output_type; - private: - typedef join_node_base , input_ports_type, output_type > base_type; - typedef typename internal::type_to_key_function_body *f0_p; - typedef typename internal::type_to_key_function_body *f1_p; - typedef typename internal::type_to_key_function_body *f2_p; - typedef typename internal::type_to_key_function_body *f3_p; - typedef typename internal::type_to_key_function_body *f4_p; - typedef typename internal::type_to_key_function_body *f5_p; - typedef typename internal::type_to_key_function_body *f6_p; - typedef typename internal::type_to_key_function_body *f7_p; - typedef typename internal::type_to_key_function_body *f8_p; - typedef typename internal::type_to_key_function_body *f9_p; - typedef typename tbb::flow::tuple< f0_p, f1_p, f2_p, f3_p, f4_p, f5_p, f6_p, f7_p, f8_p, f9_p > func_initializer_type; - public: -#if __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING - unfolded_join_node(graph &g) : base_type(g, - func_initializer_type( - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()), - new internal::type_to_key_function_body_leaf >(key_from_message_body()) - ) ) { - } -#endif /* __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING */ - template - unfolded_join_node(graph &g, Body0 body0, Body1 body1, Body2 body2, Body3 body3, Body4 body4, - Body5 body5, Body6 body6, Body7 body7, Body8 body8, Body9 body9) : base_type(g, func_initializer_type( - new internal::type_to_key_function_body_leaf(body0), - new internal::type_to_key_function_body_leaf(body1), - new internal::type_to_key_function_body_leaf(body2), - new internal::type_to_key_function_body_leaf(body3), - new internal::type_to_key_function_body_leaf(body4), - new internal::type_to_key_function_body_leaf(body5), - new internal::type_to_key_function_body_leaf(body6), - new internal::type_to_key_function_body_leaf(body7), - new internal::type_to_key_function_body_leaf(body8), - new internal::type_to_key_function_body_leaf(body9) - ) ) { - __TBB_STATIC_ASSERT(tbb::flow::tuple_size::value == 10, "wrong number of body initializers"); - } - unfolded_join_node(const unfolded_join_node &other) : base_type(other) {} - }; -#endif - - //! templated function to refer to input ports of the join node - template - typename tbb::flow::tuple_element::type &input_port(JNT &jn) { - return tbb::flow::get(jn.input_ports()); - } - -} -#endif // __TBB__flow_graph_join_impl_H - diff --git a/src/3rd party/tbb/internal/_flow_graph_node_impl.h b/src/3rd party/tbb/internal/_flow_graph_node_impl.h deleted file mode 100644 index e6277b24e5..0000000000 --- a/src/3rd party/tbb/internal/_flow_graph_node_impl.h +++ /dev/null @@ -1,971 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB__flow_graph_node_impl_H -#define __TBB__flow_graph_node_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include "_flow_graph_item_buffer_impl.h" - -//! @cond INTERNAL -namespace internal { - - using tbb::internal::aggregated_operation; - using tbb::internal::aggregating_functor; - using tbb::internal::aggregator; - - template< typename T, typename A > - class function_input_queue : public item_buffer { - public: - bool empty() const { - return this->buffer_empty(); - } - - const T& front() const { - return this->item_buffer::front(); - } - - bool pop( T& t ) { - return this->pop_front( t ); - } - - void pop() { - this->destroy_front(); - } - - bool push( T& t ) { - return this->push_back( t ); - } - }; - - //! Input and scheduling for a function node that takes a type Input as input - // The only up-ref is apply_body_impl, which should implement the function - // call and any handling of the result. - template< typename Input, typename Policy, typename A, typename ImplType > - class function_input_base : public receiver, tbb::internal::no_assign { - enum op_type {reg_pred, rem_pred, try_fwd, tryput_bypass, app_body_bypass, occupy_concurrency -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - , add_blt_pred, del_blt_pred, - blt_pred_cnt, blt_pred_cpy // create vector copies of preds and succs -#endif - }; - typedef function_input_base class_type; - - public: - - //! The input type of this receiver - typedef Input input_type; - typedef typename receiver::predecessor_type predecessor_type; - typedef predecessor_cache predecessor_cache_type; - typedef function_input_queue input_queue_type; - typedef typename A::template rebind< input_queue_type >::other queue_allocator_type; - __TBB_STATIC_ASSERT(!((internal::has_policy::value) && (internal::has_policy::value)), - "queueing and rejecting policies can't be specified simultaneously"); - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename predecessor_cache_type::built_predecessors_type built_predecessors_type; - typedef typename receiver::predecessor_list_type predecessor_list_type; -#endif - - //! Constructor for function_input_base - function_input_base( - graph &g, __TBB_FLOW_GRAPH_PRIORITY_ARG1(size_t max_concurrency, node_priority_t priority) - ) : my_graph_ref(g), my_max_concurrency(max_concurrency) - , __TBB_FLOW_GRAPH_PRIORITY_ARG1(my_concurrency(0), my_priority(priority)) - , my_queue(!internal::has_policy::value ? new input_queue_type() : NULL) - , forwarder_busy(false) - { - my_predecessors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - //! Copy constructor - function_input_base( const function_input_base& src) - : receiver(), tbb::internal::no_assign() - , my_graph_ref(src.my_graph_ref), my_max_concurrency(src.my_max_concurrency) - , __TBB_FLOW_GRAPH_PRIORITY_ARG1(my_concurrency(0), my_priority(src.my_priority)) - , my_queue(src.my_queue ? new input_queue_type() : NULL), forwarder_busy(false) - { - my_predecessors.set_owner(this); - my_aggregator.initialize_handler(handler_type(this)); - } - - //! Destructor - // The queue is allocated by the constructor for {multi}function_node. - // TODO: pass the graph_buffer_policy to the base so it can allocate the queue instead. - // This would be an interface-breaking change. - virtual ~function_input_base() { - if ( my_queue ) delete my_queue; - } - - task* try_put_task( const input_type& t) __TBB_override { - return try_put_task_impl(t, internal::has_policy()); - } - - //! Adds src to the list of cached predecessors. - bool register_predecessor( predecessor_type &src ) __TBB_override { - operation_type op_data(reg_pred); - op_data.r = &src; - my_aggregator.execute(&op_data); - return true; - } - - //! Removes src from the list of cached predecessors. - bool remove_predecessor( predecessor_type &src ) __TBB_override { - operation_type op_data(rem_pred); - op_data.r = &src; - my_aggregator.execute(&op_data); - return true; - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - //! Adds to list of predecessors added by make_edge - void internal_add_built_predecessor( predecessor_type &src) __TBB_override { - operation_type op_data(add_blt_pred); - op_data.r = &src; - my_aggregator.execute(&op_data); - } - - //! removes from to list of predecessors (used by remove_edge) - void internal_delete_built_predecessor( predecessor_type &src) __TBB_override { - operation_type op_data(del_blt_pred); - op_data.r = &src; - my_aggregator.execute(&op_data); - } - - size_t predecessor_count() __TBB_override { - operation_type op_data(blt_pred_cnt); - my_aggregator.execute(&op_data); - return op_data.cnt_val; - } - - void copy_predecessors(predecessor_list_type &v) __TBB_override { - operation_type op_data(blt_pred_cpy); - op_data.predv = &v; - my_aggregator.execute(&op_data); - } - - built_predecessors_type &built_predecessors() __TBB_override { - return my_predecessors.built_predecessors(); - } -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - - protected: - - void reset_function_input_base( reset_flags f) { - my_concurrency = 0; - if(my_queue) { - my_queue->reset(); - } - reset_receiver(f); - forwarder_busy = false; - } - - graph& my_graph_ref; - const size_t my_max_concurrency; - size_t my_concurrency; - __TBB_FLOW_GRAPH_PRIORITY_EXPR( node_priority_t my_priority; ) - input_queue_type *my_queue; - predecessor_cache my_predecessors; - - void reset_receiver( reset_flags f) __TBB_override { - if( f & rf_clear_edges) my_predecessors.clear(); - else - my_predecessors.reset(); - __TBB_ASSERT(!(f & rf_clear_edges) || my_predecessors.empty(), "function_input_base reset failed"); - } - - graph& graph_reference() const __TBB_override { - return my_graph_ref; - } - - task* try_get_postponed_task(const input_type& i) { - operation_type op_data(i, app_body_bypass); // tries to pop an item or get_item - my_aggregator.execute(&op_data); - return op_data.bypass_t; - } - - private: - - friend class apply_body_task_bypass< class_type, input_type >; - friend class forward_task_bypass< class_type >; - - class operation_type : public aggregated_operation< operation_type > { - public: - char type; - union { - input_type *elem; - predecessor_type *r; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - size_t cnt_val; - predecessor_list_type *predv; -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - }; - tbb::task *bypass_t; - operation_type(const input_type& e, op_type t) : - type(char(t)), elem(const_cast(&e)) {} - operation_type(op_type t) : type(char(t)), r(NULL) {} - }; - - bool forwarder_busy; - typedef internal::aggregating_functor handler_type; - friend class internal::aggregating_functor; - aggregator< handler_type, operation_type > my_aggregator; - - task* perform_queued_requests() { - task* new_task = NULL; - if(my_queue) { - if(!my_queue->empty()) { - ++my_concurrency; - new_task = create_body_task(my_queue->front()); - - my_queue->pop(); - } - } - else { - input_type i; - if(my_predecessors.get_item(i)) { - ++my_concurrency; - new_task = create_body_task(i); - } - } - return new_task; - } - void handle_operations(operation_type *op_list) { - operation_type *tmp; - while (op_list) { - tmp = op_list; - op_list = op_list->next; - switch (tmp->type) { - case reg_pred: - my_predecessors.add(*(tmp->r)); - __TBB_store_with_release(tmp->status, SUCCEEDED); - if (!forwarder_busy) { - forwarder_busy = true; - spawn_forward_task(); - } - break; - case rem_pred: - my_predecessors.remove(*(tmp->r)); - __TBB_store_with_release(tmp->status, SUCCEEDED); - break; - case app_body_bypass: { - tmp->bypass_t = NULL; - __TBB_ASSERT(my_max_concurrency != 0, NULL); - --my_concurrency; - if(my_concurrencybypass_t = perform_queued_requests(); - - __TBB_store_with_release(tmp->status, SUCCEEDED); - } - break; - case tryput_bypass: internal_try_put_task(tmp); break; - case try_fwd: internal_forward(tmp); break; - case occupy_concurrency: - if (my_concurrency < my_max_concurrency) { - ++my_concurrency; - __TBB_store_with_release(tmp->status, SUCCEEDED); - } else { - __TBB_store_with_release(tmp->status, FAILED); - } - break; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - case add_blt_pred: { - my_predecessors.internal_add_built_predecessor(*(tmp->r)); - __TBB_store_with_release(tmp->status, SUCCEEDED); - } - break; - case del_blt_pred: - my_predecessors.internal_delete_built_predecessor(*(tmp->r)); - __TBB_store_with_release(tmp->status, SUCCEEDED); - break; - case blt_pred_cnt: - tmp->cnt_val = my_predecessors.predecessor_count(); - __TBB_store_with_release(tmp->status, SUCCEEDED); - break; - case blt_pred_cpy: - my_predecessors.copy_predecessors( *(tmp->predv) ); - __TBB_store_with_release(tmp->status, SUCCEEDED); - break; -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - } - } - } - - //! Put to the node, but return the task instead of enqueueing it - void internal_try_put_task(operation_type *op) { - __TBB_ASSERT(my_max_concurrency != 0, NULL); - if (my_concurrency < my_max_concurrency) { - ++my_concurrency; - task * new_task = create_body_task(*(op->elem)); - op->bypass_t = new_task; - __TBB_store_with_release(op->status, SUCCEEDED); - } else if ( my_queue && my_queue->push(*(op->elem)) ) { - op->bypass_t = SUCCESSFULLY_ENQUEUED; - __TBB_store_with_release(op->status, SUCCEEDED); - } else { - op->bypass_t = NULL; - __TBB_store_with_release(op->status, FAILED); - } - } - - //! Creates tasks for postponed messages if available and if concurrency allows - void internal_forward(operation_type *op) { - op->bypass_t = NULL; - if (my_concurrency < my_max_concurrency || !my_max_concurrency) - op->bypass_t = perform_queued_requests(); - if(op->bypass_t) - __TBB_store_with_release(op->status, SUCCEEDED); - else { - forwarder_busy = false; - __TBB_store_with_release(op->status, FAILED); - } - } - - task* internal_try_put_bypass( const input_type& t ) { - operation_type op_data(t, tryput_bypass); - my_aggregator.execute(&op_data); - if( op_data.status == internal::SUCCEEDED ) { - return op_data.bypass_t; - } - return NULL; - } - - task* try_put_task_impl( const input_type& t, /*lightweight=*/tbb::internal::true_type ) { - if( my_max_concurrency == 0 ) { - return apply_body_bypass(t); - } else { - operation_type check_op(t, occupy_concurrency); - my_aggregator.execute(&check_op); - if( check_op.status == internal::SUCCEEDED ) { - return apply_body_bypass(t); - } - return internal_try_put_bypass(t); - } - } - - task* try_put_task_impl( const input_type& t, /*lightweight=*/tbb::internal::false_type ) { - if( my_max_concurrency == 0 ) { - return create_body_task(t); - } else { - return internal_try_put_bypass(t); - } - } - - //! Applies the body to the provided input - // then decides if more work is available - task * apply_body_bypass( const input_type &i ) { - return static_cast(this)->apply_body_impl_bypass(i); - } - - //! allocates a task to apply a body - inline task * create_body_task( const input_type &input ) { - return (internal::is_graph_active(my_graph_ref)) ? - new( task::allocate_additional_child_of(*(my_graph_ref.root_task())) ) - apply_body_task_bypass < class_type, input_type >( - *this, __TBB_FLOW_GRAPH_PRIORITY_ARG1(input, my_priority)) - : NULL; - } - - //! This is executed by an enqueued task, the "forwarder" - task* forward_task() { - operation_type op_data(try_fwd); - task* rval = NULL; - do { - op_data.status = WAIT; - my_aggregator.execute(&op_data); - if(op_data.status == SUCCEEDED) { - task* ttask = op_data.bypass_t; - __TBB_ASSERT( ttask && ttask != SUCCESSFULLY_ENQUEUED, NULL ); - rval = combine_tasks(my_graph_ref, rval, ttask); - } - } while (op_data.status == SUCCEEDED); - return rval; - } - - inline task *create_forward_task() { - return (internal::is_graph_active(my_graph_ref)) ? - new( task::allocate_additional_child_of(*(my_graph_ref.root_task())) ) - forward_task_bypass< class_type >( __TBB_FLOW_GRAPH_PRIORITY_ARG1(*this, my_priority) ) - : NULL; - } - - //! Spawns a task that calls forward() - inline void spawn_forward_task() { - task* tp = create_forward_task(); - if(tp) { - internal::spawn_in_graph_arena(graph_reference(), *tp); - } - } - }; // function_input_base - - //! Implements methods for a function node that takes a type Input as input and sends - // a type Output to its successors. - template< typename Input, typename Output, typename Policy, typename A> - class function_input : public function_input_base > { - public: - typedef Input input_type; - typedef Output output_type; - typedef function_body function_body_type; - typedef function_input my_class; - typedef function_input_base base_type; - typedef function_input_queue input_queue_type; - - // constructor - template - function_input( - graph &g, size_t max_concurrency, - __TBB_FLOW_GRAPH_PRIORITY_ARG1(Body& body, node_priority_t priority) - ) : base_type(g, __TBB_FLOW_GRAPH_PRIORITY_ARG1(max_concurrency, priority)) - , my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) - , my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) { - } - - //! Copy constructor - function_input( const function_input& src ) : - base_type(src), - my_body( src.my_init_body->clone() ), - my_init_body(src.my_init_body->clone() ) { - } - - ~function_input() { - delete my_body; - delete my_init_body; - } - - template< typename Body > - Body copy_function_object() { - function_body_type &body_ref = *this->my_body; - return dynamic_cast< internal::function_body_leaf & >(body_ref).get_body(); - } - - output_type apply_body_impl( const input_type& i) { - // There is an extra copied needed to capture the - // body execution without the try_put - tbb::internal::fgt_begin_body( my_body ); - output_type v = (*my_body)(i); - tbb::internal::fgt_end_body( my_body ); - return v; - } - - //TODO: consider moving into the base class - task * apply_body_impl_bypass( const input_type &i) { - output_type v = apply_body_impl(i); -#if TBB_DEPRECATED_MESSAGE_FLOW_ORDER - task* successor_task = successors().try_put_task(v); -#endif - task* postponed_task = NULL; - if( base_type::my_max_concurrency != 0 ) { - postponed_task = base_type::try_get_postponed_task(i); - __TBB_ASSERT( !postponed_task || postponed_task != SUCCESSFULLY_ENQUEUED, NULL ); - } -#if TBB_DEPRECATED_MESSAGE_FLOW_ORDER - graph& g = base_type::my_graph_ref; - return combine_tasks(g, successor_task, postponed_task); -#else - if( postponed_task ) { - // make the task available for other workers since we do not know successors' - // execution policy - internal::spawn_in_graph_arena(base_type::graph_reference(), *postponed_task); - } - task* successor_task = successors().try_put_task(v); -#if _MSC_VER && !__INTEL_COMPILER -#pragma warning (push) -#pragma warning (disable: 4127) /* suppress conditional expression is constant */ -#endif - if(internal::has_policy::value) { -#if _MSC_VER && !__INTEL_COMPILER -#pragma warning (pop) -#endif - if(!successor_task) { - // Return confirmative status since current - // node's body has been executed anyway - successor_task = SUCCESSFULLY_ENQUEUED; - } - } - return successor_task; -#endif /* TBB_DEPRECATED_MESSAGE_FLOW_ORDER */ - } - - protected: - - void reset_function_input(reset_flags f) { - base_type::reset_function_input_base(f); - if(f & rf_reset_bodies) { - function_body_type *tmp = my_init_body->clone(); - delete my_body; - my_body = tmp; - } - } - - function_body_type *my_body; - function_body_type *my_init_body; - virtual broadcast_cache &successors() = 0; - - }; // function_input - - - // helper templates to clear the successor edges of the output ports of an multifunction_node - template struct clear_element { - template static void clear_this(P &p) { - (void)tbb::flow::get(p).successors().clear(); - clear_element::clear_this(p); - } - template static bool this_empty(P &p) { - if(tbb::flow::get(p).successors().empty()) - return clear_element::this_empty(p); - return false; - } - }; - - template<> struct clear_element<1> { - template static void clear_this(P &p) { - (void)tbb::flow::get<0>(p).successors().clear(); - } - template static bool this_empty(P &p) { - return tbb::flow::get<0>(p).successors().empty(); - } - }; - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - // helper templates to extract the output ports of an multifunction_node from graph - template struct extract_element { - template static void extract_this(P &p) { - (void)tbb::flow::get(p).successors().built_successors().sender_extract(tbb::flow::get(p)); - extract_element::extract_this(p); - } - }; - - template<> struct extract_element<1> { - template static void extract_this(P &p) { - (void)tbb::flow::get<0>(p).successors().built_successors().sender_extract(tbb::flow::get<0>(p)); - } - }; -#endif - - template - struct init_output_ports { -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - static OutputTuple call(graph& g, const tbb::flow::tuple&) { - return OutputTuple(Args(g)...); - } -#else // __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - static OutputTuple call(graph& g, const tbb::flow::tuple&) { - return OutputTuple(T1(g)); - } - - template - static OutputTuple call(graph& g, const tbb::flow::tuple&) { - return OutputTuple(T1(g), T2(g)); - } - - template - static OutputTuple call(graph& g, const tbb::flow::tuple&) { - return OutputTuple(T1(g), T2(g), T3(g)); - } - - template - static OutputTuple call(graph& g, const tbb::flow::tuple&) { - return OutputTuple(T1(g), T2(g), T3(g), T4(g)); - } - - template - static OutputTuple call(graph& g, const tbb::flow::tuple&) { - return OutputTuple(T1(g), T2(g), T3(g), T4(g), T5(g)); - } -#if __TBB_VARIADIC_MAX >= 6 - template - static OutputTuple call(graph& g, const tbb::flow::tuple&) { - return OutputTuple(T1(g), T2(g), T3(g), T4(g), T5(g), T6(g)); - } -#endif -#if __TBB_VARIADIC_MAX >= 7 - template - static OutputTuple call(graph& g, - const tbb::flow::tuple&) { - return OutputTuple(T1(g), T2(g), T3(g), T4(g), T5(g), T6(g), T7(g)); - } -#endif -#if __TBB_VARIADIC_MAX >= 8 - template - static OutputTuple call(graph& g, - const tbb::flow::tuple&) { - return OutputTuple(T1(g), T2(g), T3(g), T4(g), T5(g), T6(g), T7(g), T8(g)); - } -#endif -#if __TBB_VARIADIC_MAX >= 9 - template - static OutputTuple call(graph& g, - const tbb::flow::tuple&) { - return OutputTuple(T1(g), T2(g), T3(g), T4(g), T5(g), T6(g), T7(g), T8(g), T9(g)); - } -#endif -#if __TBB_VARIADIC_MAX >= 9 - template - static OutputTuple call(graph& g, - const tbb::flow::tuple&) { - return OutputTuple(T1(g), T2(g), T3(g), T4(g), T5(g), T6(g), T7(g), T8(g), T9(g), T10(g)); - } -#endif -#endif // __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - }; // struct init_output_ports - - //! Implements methods for a function node that takes a type Input as input - // and has a tuple of output ports specified. - template< typename Input, typename OutputPortSet, typename Policy, typename A> - class multifunction_input : public function_input_base > { - public: - static const int N = tbb::flow::tuple_size::value; - typedef Input input_type; - typedef OutputPortSet output_ports_type; - typedef multifunction_body multifunction_body_type; - typedef multifunction_input my_class; - typedef function_input_base base_type; - typedef function_input_queue input_queue_type; - - // constructor - template - multifunction_input(graph &g, size_t max_concurrency, - __TBB_FLOW_GRAPH_PRIORITY_ARG1(Body& body, node_priority_t priority) - ) : base_type(g, __TBB_FLOW_GRAPH_PRIORITY_ARG1(max_concurrency, priority)) - , my_body( new internal::multifunction_body_leaf(body) ) - , my_init_body( new internal::multifunction_body_leaf(body) ) - , my_output_ports(init_output_ports::call(g, my_output_ports)){ - } - - //! Copy constructor - multifunction_input( const multifunction_input& src ) : - base_type(src), - my_body( src.my_init_body->clone() ), - my_init_body(src.my_init_body->clone() ), - my_output_ports( init_output_ports::call(src.my_graph_ref, my_output_ports) ) { - } - - ~multifunction_input() { - delete my_body; - delete my_init_body; - } - - template< typename Body > - Body copy_function_object() { - multifunction_body_type &body_ref = *this->my_body; - return *static_cast(dynamic_cast< internal::multifunction_body_leaf & >(body_ref).get_body_ptr()); - } - - // for multifunction nodes we do not have a single successor as such. So we just tell - // the task we were successful. - //TODO: consider moving common parts with implementation in function_input into separate function - task * apply_body_impl_bypass( const input_type &i) { - tbb::internal::fgt_begin_body( my_body ); - (*my_body)(i, my_output_ports); - tbb::internal::fgt_end_body( my_body ); - task* ttask = NULL; - if(base_type::my_max_concurrency != 0) { - ttask = base_type::try_get_postponed_task(i); - } - return ttask ? ttask : SUCCESSFULLY_ENQUEUED; - } - - output_ports_type &output_ports(){ return my_output_ports; } - - protected: -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - void extract() { - extract_element::extract_this(my_output_ports); - } -#endif - - void reset(reset_flags f) { - base_type::reset_function_input_base(f); - if(f & rf_clear_edges)clear_element::clear_this(my_output_ports); - if(f & rf_reset_bodies) { - multifunction_body_type *tmp = my_init_body->clone(); - delete my_body; - my_body = tmp; - } - __TBB_ASSERT(!(f & rf_clear_edges) || clear_element::this_empty(my_output_ports), "multifunction_node reset failed"); - } - - multifunction_body_type *my_body; - multifunction_body_type *my_init_body; - output_ports_type my_output_ports; - - }; // multifunction_input - - // template to refer to an output port of a multifunction_node - template - typename tbb::flow::tuple_element::type &output_port(MOP &op) { - return tbb::flow::get(op.output_ports()); - } - - inline void check_task_and_spawn(graph& g, task* t) { - if (t && t != SUCCESSFULLY_ENQUEUED) { - internal::spawn_in_graph_arena(g, *t); - } - } - - // helper structs for split_node - template - struct emit_element { - template - static task* emit_this(graph& g, const T &t, P &p) { - // TODO: consider to collect all the tasks in task_list and spawn them all at once - task* last_task = tbb::flow::get(p).try_put_task(tbb::flow::get(t)); - check_task_and_spawn(g, last_task); - return emit_element::emit_this(g,t,p); - } - }; - - template<> - struct emit_element<1> { - template - static task* emit_this(graph& g, const T &t, P &p) { - task* last_task = tbb::flow::get<0>(p).try_put_task(tbb::flow::get<0>(t)); - check_task_and_spawn(g, last_task); - return SUCCESSFULLY_ENQUEUED; - } - }; - - //! Implements methods for an executable node that takes continue_msg as input - template< typename Output, typename Policy> - class continue_input : public continue_receiver { - public: - - //! The input type of this receiver - typedef continue_msg input_type; - - //! The output type of this receiver - typedef Output output_type; - typedef function_body function_body_type; - typedef continue_input class_type; - - template< typename Body > - continue_input( graph &g, __TBB_FLOW_GRAPH_PRIORITY_ARG1(Body& body, node_priority_t priority) ) - : continue_receiver(__TBB_FLOW_GRAPH_PRIORITY_ARG1(/*number_of_predecessors=*/0, priority)) - , my_graph_ref(g) - , my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) - , my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) - { } - - template< typename Body > - continue_input( graph &g, int number_of_predecessors, - __TBB_FLOW_GRAPH_PRIORITY_ARG1(Body& body, node_priority_t priority) - ) : continue_receiver( __TBB_FLOW_GRAPH_PRIORITY_ARG1(number_of_predecessors, priority) ) - , my_graph_ref(g) - , my_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) - , my_init_body( new internal::function_body_leaf< input_type, output_type, Body>(body) ) - { } - - continue_input( const continue_input& src ) : continue_receiver(src), - my_graph_ref(src.my_graph_ref), - my_body( src.my_init_body->clone() ), - my_init_body( src.my_init_body->clone() ) {} - - ~continue_input() { - delete my_body; - delete my_init_body; - } - - template< typename Body > - Body copy_function_object() { - function_body_type &body_ref = *my_body; - return dynamic_cast< internal::function_body_leaf & >(body_ref).get_body(); - } - - void reset_receiver( reset_flags f) __TBB_override { - continue_receiver::reset_receiver(f); - if(f & rf_reset_bodies) { - function_body_type *tmp = my_init_body->clone(); - delete my_body; - my_body = tmp; - } - } - - protected: - - graph& my_graph_ref; - function_body_type *my_body; - function_body_type *my_init_body; - - virtual broadcast_cache &successors() = 0; - - friend class apply_body_task_bypass< class_type, continue_msg >; - - //! Applies the body to the provided input - task *apply_body_bypass( input_type ) { - // There is an extra copied needed to capture the - // body execution without the try_put - tbb::internal::fgt_begin_body( my_body ); - output_type v = (*my_body)( continue_msg() ); - tbb::internal::fgt_end_body( my_body ); - return successors().try_put_task( v ); - } - - task* execute() __TBB_override { - if(!internal::is_graph_active(my_graph_ref)) { - return NULL; - } -#if _MSC_VER && !__INTEL_COMPILER -#pragma warning (push) -#pragma warning (disable: 4127) /* suppress conditional expression is constant */ -#endif - if(internal::has_policy::value) { -#if _MSC_VER && !__INTEL_COMPILER -#pragma warning (pop) -#endif - return apply_body_bypass( continue_msg() ); - } - else { - return new ( task::allocate_additional_child_of( *(my_graph_ref.root_task()) ) ) - apply_body_task_bypass< class_type, continue_msg >( - *this, __TBB_FLOW_GRAPH_PRIORITY_ARG1(continue_msg(), my_priority) ); - } - } - - graph& graph_reference() const __TBB_override { - return my_graph_ref; - } - }; // continue_input - - //! Implements methods for both executable and function nodes that puts Output to its successors - template< typename Output > - class function_output : public sender { - public: - - template friend struct clear_element; - typedef Output output_type; - typedef typename sender::successor_type successor_type; - typedef broadcast_cache broadcast_cache_type; -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - typedef typename sender::built_successors_type built_successors_type; - typedef typename sender::successor_list_type successor_list_type; -#endif - - function_output( graph& g) : my_graph_ref(g) { my_successors.set_owner(this); } - function_output(const function_output & other) : sender(), my_graph_ref(other.my_graph_ref) { - my_successors.set_owner(this); - } - - //! Adds a new successor to this node - bool register_successor( successor_type &r ) __TBB_override { - successors().register_successor( r ); - return true; - } - - //! Removes a successor from this node - bool remove_successor( successor_type &r ) __TBB_override { - successors().remove_successor( r ); - return true; - } - -#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION - built_successors_type &built_successors() __TBB_override { return successors().built_successors(); } - - - void internal_add_built_successor( successor_type &r) __TBB_override { - successors().internal_add_built_successor( r ); - } - - void internal_delete_built_successor( successor_type &r) __TBB_override { - successors().internal_delete_built_successor( r ); - } - - size_t successor_count() __TBB_override { - return successors().successor_count(); - } - - void copy_successors( successor_list_type &v) __TBB_override { - successors().copy_successors(v); - } -#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ - - // for multifunction_node. The function_body that implements - // the node will have an input and an output tuple of ports. To put - // an item to a successor, the body should - // - // get(output_ports).try_put(output_value); - // - // if task pointer is returned will always spawn and return true, else - // return value will be bool returned from successors.try_put. - task *try_put_task(const output_type &i) { // not a virtual method in this class - return my_successors.try_put_task(i); - } - - broadcast_cache_type &successors() { return my_successors; } - - graph& graph_reference() const { return my_graph_ref; } - protected: - broadcast_cache_type my_successors; - graph& my_graph_ref; - }; // function_output - - template< typename Output > - class multifunction_output : public function_output { - public: - typedef Output output_type; - typedef function_output base_type; - using base_type::my_successors; - - multifunction_output(graph& g) : base_type(g) {my_successors.set_owner(this);} - multifunction_output( const multifunction_output& other) : base_type(other.my_graph_ref) { my_successors.set_owner(this); } - - bool try_put(const output_type &i) { - task *res = try_put_task(i); - if(!res) return false; - if(res != SUCCESSFULLY_ENQUEUED) { - FLOW_SPAWN(*res); // TODO: Spawn task inside arena - } - return true; - } - - using base_type::graph_reference; - - protected: - - task* try_put_task(const output_type &i) { - return my_successors.try_put_task(i); - } - - template friend struct emit_element; - - }; // multifunction_output - -//composite_node -#if __TBB_FLOW_GRAPH_CPP11_FEATURES - template - void add_nodes_impl(CompositeType*, bool) {} - - template< typename CompositeType, typename NodeType1, typename... NodeTypes > - void add_nodes_impl(CompositeType *c_node, bool visible, const NodeType1& n1, const NodeTypes&... n) { - void *addr = const_cast(&n1); - - fgt_alias_port(c_node, addr, visible); - add_nodes_impl(c_node, visible, n...); - } -#endif - -} // internal - -#endif // __TBB__flow_graph_node_impl_H diff --git a/src/3rd party/tbb/internal/_flow_graph_nodes_deduction.h b/src/3rd party/tbb/internal/_flow_graph_nodes_deduction.h deleted file mode 100644 index c12553fdd6..0000000000 --- a/src/3rd party/tbb/internal/_flow_graph_nodes_deduction.h +++ /dev/null @@ -1,260 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_flow_graph_nodes_deduction_H -#define __TBB_flow_graph_nodes_deduction_H - -#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT - -namespace tbb { -namespace flow { -namespace interface11 { - -template -struct declare_body_types { - using input_type = Input; - using output_type = Output; -}; - -template struct body_types; - -template -struct body_types : declare_body_types {}; - -template -struct body_types : declare_body_types {}; - -template -struct body_types : declare_body_types {}; - -template -struct body_types : declare_body_types {}; - -template -struct body_types : declare_body_types {}; - -template -struct body_types : declare_body_types {}; - -template -using input_t = typename body_types::input_type; - -template -using output_t = typename body_types::output_type; - -template -auto decide_on_operator_overload(Output (T::*name)(const Input&) const)->decltype(name); - -template -auto decide_on_operator_overload(Output (T::*name)(const Input&))->decltype(name); - -template -auto decide_on_operator_overload(Output (T::*name)(Input&) const)->decltype(name); - -template -auto decide_on_operator_overload(Output (T::*name)(Input&))->decltype(name); - -template -auto decide_on_operator_overload(Output (*name)(const Input&))->decltype(name); - -template -auto decide_on_operator_overload(Output (*name)(Input&))->decltype(name); - -template -decltype(decide_on_operator_overload(&Body::operator())) decide_on_callable_type(int); - -template -decltype(decide_on_operator_overload(std::declval())) decide_on_callable_type(...); - -// Deduction guides for Flow Graph nodes -template -source_node(GraphOrSet&&, Body, bool = true) -->source_node(0))>>; - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - -template -struct decide_on_set; - -template -struct decide_on_set> { - using type = typename Node::output_type; -}; - -template -struct decide_on_set> { - using type = typename Node::input_type; -}; - -template -using decide_on_set_t = typename decide_on_set>::type; - -template -broadcast_node(const NodeSet&) -->broadcast_node>; - -template -buffer_node(const NodeSet&) -->buffer_node>; - -template -queue_node(const NodeSet&) -->queue_node>; -#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - -template -sequencer_node(GraphOrProxy&&, Sequencer) -->sequencer_node(0))>>; - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET -template -priority_queue_node(const NodeSet&, const Compare&) -->priority_queue_node, Compare>; - -template -priority_queue_node(const NodeSet&) -->priority_queue_node, std::less>>; -#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - -template -struct join_key { - using type = Key; -}; - -template -struct join_key { - using type = T&; -}; - -template -using join_key_t = typename join_key::type; - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET -template -join_node(const node_set&, Policy) -->join_node, - Policy>; - -template -join_node(const node_set&, Policy) -->join_node; - -template -join_node(const node_set) -->join_node, - queueing>; - -template -join_node(const node_set) -->join_node; -#endif - -template -join_node(GraphOrProxy&&, Body, Bodies...) -->join_node(0))>, - input_t(0))>...>, - key_matching(0))>>>>; - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET -template -indexer_node(const node_set&) -->indexer_node; -#endif - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET -template -limiter_node(const NodeSet&, size_t) -->limiter_node>; - -template -split_node(const node_set&) -->split_node; - -template -split_node(const node_set&) -->split_node>; - -#endif - -template -function_node(GraphOrSet&&, - size_t, Body, - __TBB_FLOW_GRAPH_PRIORITY_ARG1(Policy, node_priority_t = tbb::flow::internal::no_priority)) -->function_node(0))>, - output_t(0))>, - Policy>; - -template -function_node(GraphOrSet&&, size_t, - __TBB_FLOW_GRAPH_PRIORITY_ARG1(Body, node_priority_t = tbb::flow::internal::no_priority)) -->function_node(0))>, - output_t(0))>, - queueing>; - -template -struct continue_output { - using type = Output; -}; - -template <> -struct continue_output { - using type = continue_msg; -}; - -template -using continue_output_t = typename continue_output::type; - -template -continue_node(GraphOrSet&&, Body, - __TBB_FLOW_GRAPH_PRIORITY_ARG1(Policy, node_priority_t = tbb::flow::internal::no_priority)) -->continue_node>, - Policy>; - -template -continue_node(GraphOrSet&&, - int, Body, - __TBB_FLOW_GRAPH_PRIORITY_ARG1(Policy, node_priority_t = tbb::flow::internal::no_priority)) -->continue_node>, - Policy>; - -template -continue_node(GraphOrSet&&, - __TBB_FLOW_GRAPH_PRIORITY_ARG1(Body, node_priority_t = tbb::flow::internal::no_priority)) -->continue_node>, - internal::Policy>; - -template -continue_node(GraphOrSet&&, int, - __TBB_FLOW_GRAPH_PRIORITY_ARG1(Body, node_priority_t = tbb::flow::internal::no_priority)) -->continue_node>, - internal::Policy>; - -#if __TBB_PREVIEW_FLOW_GRAPH_NODE_SET - -template -overwrite_node(const NodeSet&) -->overwrite_node>; - -template -write_once_node(const NodeSet&) -->write_once_node>; -#endif // __TBB_PREVIEW_FLOW_GRAPH_NODE_SET -} // namespace interfaceX -} // namespace flow -} // namespace tbb - -#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT -#endif // __TBB_flow_graph_nodes_deduction_H diff --git a/src/3rd party/tbb/internal/_flow_graph_streaming_node.h b/src/3rd party/tbb/internal/_flow_graph_streaming_node.h deleted file mode 100644 index 4fb19c7a32..0000000000 --- a/src/3rd party/tbb/internal/_flow_graph_streaming_node.h +++ /dev/null @@ -1,742 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_flow_graph_streaming_H -#define __TBB_flow_graph_streaming_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#if __TBB_PREVIEW_STREAMING_NODE - -// Included in namespace tbb::flow::interfaceX (in flow_graph.h) - -namespace internal { - -template -struct port_ref_impl { - // "+1" since the port_ref range is a closed interval (includes its endpoints). - static const int size = N2 - N1 + 1; -}; - -} // internal - -// The purpose of the port_ref_impl is the pretty syntax: the deduction of a compile-time constant is processed from the return type. -// So it is possible to use this helper without parentheses, e.g. "port_ref<0>". -template -internal::port_ref_impl port_ref() { - return internal::port_ref_impl(); -}; - -namespace internal { - -template -struct num_arguments { - static const int value = 1; -}; - -template -struct num_arguments(*)()> { - static const int value = port_ref_impl::size; -}; - -template -struct num_arguments> { - static const int value = port_ref_impl::size; -}; - -template -void ignore_return_values( Args&&... ) {} - -template -T or_return_values( T&& t ) { return t; } -template -T or_return_values( T&& t, Rest&&... rest ) { - return t | or_return_values( std::forward(rest)... ); -} - -template -struct key_from_policy { - typedef size_t type; - typedef std::false_type is_key_matching; -}; - -template -struct key_from_policy< key_matching > { - typedef Key type; - typedef std::true_type is_key_matching; -}; - -template -struct key_from_policy< key_matching > { - typedef const Key &type; - typedef std::true_type is_key_matching; -}; - -template -class streaming_device_with_key { - Device my_device; - typename std::decay::type my_key; -public: - // TODO: investigate why default constructor is required - streaming_device_with_key() {} - streaming_device_with_key( const Device& d, Key k ) : my_device( d ), my_key( k ) {} - Key key() const { return my_key; } - const Device& device() const { return my_device; } -}; - -// --------- Kernel argument helpers --------- // -template -struct is_port_ref_impl { - typedef std::false_type type; -}; - -template -struct is_port_ref_impl< port_ref_impl > { - typedef std::true_type type; -}; - -template -struct is_port_ref_impl< port_ref_impl( * )() > { - typedef std::true_type type; -}; - -template -struct is_port_ref { - typedef typename is_port_ref_impl< typename tbb::internal::strip::type >::type type; -}; - -template -struct convert_and_call_impl; - -template -struct convert_and_call_impl { - static const size_t my_delta = 1; // Index 0 contains device - - template - static void doit(F& f, Tuple& t, A1& a1, Args1&... args1, Args2&... args2) { - convert_and_call_impl::doit_impl(typename is_port_ref::type(), f, t, a1, args1..., args2...); - } - template - static void doit_impl(std::false_type, F& f, Tuple& t, A1& a1, Args1&... args1, Args2&... args2) { - convert_and_call_impl::doit(f, t, args1..., args2..., a1); - } - template - static void doit_impl(std::true_type x, F& f, Tuple& t, port_ref_impl, Args1&... args1, Args2&... args2) { - convert_and_call_impl, Args1...>::doit_impl(x, f, t, port_ref(), args1..., - args2..., std::get(t)); - } - template - static void doit_impl(std::true_type, F& f, Tuple& t, port_ref_impl, Args1&... args1, Args2&... args2) { - convert_and_call_impl::doit(f, t, args1..., args2..., std::get(t)); - } - - template - static void doit_impl(std::true_type x, F& f, Tuple& t, port_ref_impl(* fn)(), Args1&... args1, Args2&... args2) { - doit_impl(x, f, t, fn(), args1..., args2...); - } - template - static void doit_impl(std::true_type x, F& f, Tuple& t, port_ref_impl(* fn)(), Args1&... args1, Args2&... args2) { - doit_impl(x, f, t, fn(), args1..., args2...); - } -}; - -template <> -struct convert_and_call_impl<> { - template - static void doit(F& f, Tuple&, Args2&... args2) { - f(args2...); - } -}; -// ------------------------------------------- // - -template -struct streaming_node_traits { - // Do not use 'using' instead of 'struct' because Microsoft Visual C++ 12.0 fails to compile. - template - struct async_msg_type { - typedef typename StreamFactory::template async_msg_type type; - }; - - typedef tuple< typename async_msg_type::type... > input_tuple; - typedef input_tuple output_tuple; - typedef tuple< streaming_device_with_key< typename StreamFactory::device_type, typename key_from_policy::type >, - typename async_msg_type::type... > kernel_input_tuple; - - // indexer_node parameters pack expansion workaround for VS2013 for streaming_node - typedef indexer_node< typename async_msg_type::type... > indexer_node_type; -}; - -// Default empty implementation -template -class kernel_executor_helper { - typedef typename StreamFactory::device_type device_type; - typedef typename StreamFactory::kernel_type kernel_type; - typedef KernelInputTuple kernel_input_tuple; -protected: - template - void enqueue_kernel_impl( kernel_input_tuple&, StreamFactory& factory, device_type device, const kernel_type& kernel, Args&... args ) const { - factory.send_kernel( device, kernel, args... ); - } -}; - -// Implementation for StreamFactory supporting range -template -class kernel_executor_helper::type > { - typedef typename StreamFactory::device_type device_type; - typedef typename StreamFactory::kernel_type kernel_type; - typedef KernelInputTuple kernel_input_tuple; - - typedef typename StreamFactory::range_type range_type; - - // Container for randge. It can contain either port references or real range. - struct range_wrapper { - virtual range_type get_range( const kernel_input_tuple &ip ) const = 0; - virtual range_wrapper *clone() const = 0; - virtual ~range_wrapper() {} - }; - - struct range_value : public range_wrapper { - range_value( const range_type& value ) : my_value(value) {} - - range_value( range_type&& value ) : my_value(std::move(value)) {} - - range_type get_range( const kernel_input_tuple & ) const __TBB_override { - return my_value; - } - - range_wrapper *clone() const __TBB_override { - return new range_value(my_value); - } - private: - range_type my_value; - }; - - template - struct range_mapper : public range_wrapper { - range_mapper() {} - - range_type get_range( const kernel_input_tuple &ip ) const __TBB_override { - // "+1" since get<0>(ip) is StreamFactory::device. - return get(ip).data(false); - } - - range_wrapper *clone() const __TBB_override { - return new range_mapper; - } - }; - -protected: - template - void enqueue_kernel_impl( kernel_input_tuple& ip, StreamFactory& factory, device_type device, const kernel_type& kernel, Args&... args ) const { - __TBB_ASSERT(my_range_wrapper, "Range is not set. Call set_range() before running streaming_node."); - factory.send_kernel( device, kernel, my_range_wrapper->get_range(ip), args... ); - } - -public: - kernel_executor_helper() : my_range_wrapper(NULL) {} - - kernel_executor_helper(const kernel_executor_helper& executor) : my_range_wrapper(executor.my_range_wrapper ? executor.my_range_wrapper->clone() : NULL) {} - - kernel_executor_helper(kernel_executor_helper&& executor) : my_range_wrapper(executor.my_range_wrapper) { - // Set moving holder mappers to NULL to prevent double deallocation - executor.my_range_wrapper = NULL; - } - - ~kernel_executor_helper() { - if (my_range_wrapper) delete my_range_wrapper; - } - - void set_range(const range_type& work_size) { - my_range_wrapper = new range_value(work_size); - } - - void set_range(range_type&& work_size) { - my_range_wrapper = new range_value(std::move(work_size)); - } - - template - void set_range(port_ref_impl) { - my_range_wrapper = new range_mapper; - } - - template - void set_range(port_ref_impl(*)()) { - my_range_wrapper = new range_mapper; - } - -private: - range_wrapper* my_range_wrapper; -}; - -} // internal - -/* -/---------------------------------------- streaming_node ------------------------------------\ -| | -| /--------------\ /----------------------\ /-----------\ /----------------------\ | -| | | | (device_with_key) O---O | | | | -| | | | | | | | | | -O---O indexer_node O---O device_selector_node O---O join_node O---O kernel_node O---O -| | | | (multifunction_node) | | | | (multifunction_node) | | -O---O | | O---O | | O---O -| \--------------/ \----------------------/ \-----------/ \----------------------/ | -| | -\--------------------------------------------------------------------------------------------/ -*/ -template -class streaming_node; - -template -class streaming_node< tuple, JP, StreamFactory > - : public composite_node < typename internal::streaming_node_traits::input_tuple, - typename internal::streaming_node_traits::output_tuple > - , public internal::kernel_executor_helper< StreamFactory, typename internal::streaming_node_traits::kernel_input_tuple > -{ - typedef typename internal::streaming_node_traits::input_tuple input_tuple; - typedef typename internal::streaming_node_traits::output_tuple output_tuple; - typedef typename internal::key_from_policy::type key_type; -protected: - typedef typename StreamFactory::device_type device_type; - typedef typename StreamFactory::kernel_type kernel_type; -private: - typedef internal::streaming_device_with_key device_with_key_type; - typedef composite_node base_type; - static const size_t NUM_INPUTS = tuple_size::value; - static const size_t NUM_OUTPUTS = tuple_size::value; - - typedef typename internal::make_sequence::type input_sequence; - typedef typename internal::make_sequence::type output_sequence; - - typedef typename internal::streaming_node_traits::indexer_node_type indexer_node_type; - typedef typename indexer_node_type::output_type indexer_node_output_type; - typedef typename internal::streaming_node_traits::kernel_input_tuple kernel_input_tuple; - typedef multifunction_node device_selector_node; - typedef multifunction_node kernel_multifunction_node; - - template - typename base_type::input_ports_type get_input_ports( internal::sequence ) { - return std::tie( internal::input_port( my_indexer_node )... ); - } - - template - typename base_type::output_ports_type get_output_ports( internal::sequence ) { - return std::tie( internal::output_port( my_kernel_node )... ); - } - - typename base_type::input_ports_type get_input_ports() { - return get_input_ports( input_sequence() ); - } - - typename base_type::output_ports_type get_output_ports() { - return get_output_ports( output_sequence() ); - } - - template - int make_Nth_edge() { - make_edge( internal::output_port( my_device_selector_node ), internal::input_port( my_join_node ) ); - return 0; - } - - template - void make_edges( internal::sequence ) { - make_edge( my_indexer_node, my_device_selector_node ); - make_edge( my_device_selector_node, my_join_node ); - internal::ignore_return_values( make_Nth_edge()... ); - make_edge( my_join_node, my_kernel_node ); - } - - void make_edges() { - make_edges( input_sequence() ); - } - - class device_selector_base { - public: - virtual void operator()( const indexer_node_output_type &v, typename device_selector_node::output_ports_type &op ) = 0; - virtual device_selector_base *clone( streaming_node &n ) const = 0; - virtual ~device_selector_base() {} - }; - - template - class device_selector : public device_selector_base, tbb::internal::no_assign { - public: - device_selector( UserFunctor uf, streaming_node &n, StreamFactory &f ) - : my_dispatch_funcs( create_dispatch_funcs( input_sequence() ) ) - , my_user_functor( uf ), my_node(n), my_factory( f ) - { - my_port_epoches.fill( 0 ); - } - - void operator()( const indexer_node_output_type &v, typename device_selector_node::output_ports_type &op ) __TBB_override { - (this->*my_dispatch_funcs[ v.tag() ])( my_port_epoches[ v.tag() ], v, op ); - __TBB_ASSERT( (tbb::internal::is_same_type::is_key_matching, std::false_type>::value) - || my_port_epoches[v.tag()] == 0, "Epoch is changed when key matching is requested" ); - } - - device_selector_base *clone( streaming_node &n ) const __TBB_override { - return new device_selector( my_user_functor, n, my_factory ); - } - private: - typedef void(device_selector::*send_and_put_fn_type)(size_t &, const indexer_node_output_type &, typename device_selector_node::output_ports_type &); - typedef std::array < send_and_put_fn_type, NUM_INPUTS > dispatch_funcs_type; - - template - static dispatch_funcs_type create_dispatch_funcs( internal::sequence ) { - dispatch_funcs_type dispatch = { { &device_selector::send_and_put_impl... } }; - return dispatch; - } - - template - key_type get_key( std::false_type, const T &, size_t &epoch ) { - __TBB_STATIC_ASSERT( (tbb::internal::is_same_type::value), "" ); - return epoch++; - } - - template - key_type get_key( std::true_type, const T &t, size_t &/*epoch*/ ) { - using tbb::flow::key_from_message; - return key_from_message( t ); - } - - template - void send_and_put_impl( size_t &epoch, const indexer_node_output_type &v, typename device_selector_node::output_ports_type &op ) { - typedef typename tuple_element::type::output_type elem_type; - elem_type e = internal::cast_to( v ); - device_type device = get_device( get_key( typename internal::key_from_policy::is_key_matching(), e, epoch ), get<0>( op ) ); - my_factory.send_data( device, e ); - get( op ).try_put( e ); - } - - template< typename DevicePort > - device_type get_device( key_type key, DevicePort& dp ) { - typename std::unordered_map::type, epoch_desc>::iterator it = my_devices.find( key ); - if ( it == my_devices.end() ) { - device_type d = my_user_functor( my_factory ); - std::tie( it, std::ignore ) = my_devices.insert( std::make_pair( key, d ) ); - bool res = dp.try_put( device_with_key_type( d, key ) ); - __TBB_ASSERT_EX( res, NULL ); - my_node.notify_new_device( d ); - } - epoch_desc &e = it->second; - device_type d = e.my_device; - if ( ++e.my_request_number == NUM_INPUTS ) my_devices.erase( it ); - return d; - } - - struct epoch_desc { - epoch_desc(device_type d ) : my_device( d ), my_request_number( 0 ) {} - device_type my_device; - size_t my_request_number; - }; - - std::unordered_map::type, epoch_desc> my_devices; - std::array my_port_epoches; - dispatch_funcs_type my_dispatch_funcs; - UserFunctor my_user_functor; - streaming_node &my_node; - StreamFactory &my_factory; - }; - - class device_selector_body { - public: - device_selector_body( device_selector_base *d ) : my_device_selector( d ) {} - - void operator()( const indexer_node_output_type &v, typename device_selector_node::output_ports_type &op ) { - (*my_device_selector)(v, op); - } - private: - device_selector_base *my_device_selector; - }; - - // TODO: investigate why copy-construction is disallowed - class args_storage_base : tbb::internal::no_copy { - public: - typedef typename kernel_multifunction_node::output_ports_type output_ports_type; - - virtual void enqueue( kernel_input_tuple &ip, output_ports_type &op, const streaming_node &n ) = 0; - virtual void send( device_type d ) = 0; - virtual args_storage_base *clone() const = 0; - virtual ~args_storage_base () {} - - protected: - args_storage_base( const kernel_type& kernel, StreamFactory &f ) - : my_kernel( kernel ), my_factory( f ) - {} - - args_storage_base( const args_storage_base &k ) - : tbb::internal::no_copy(), my_kernel( k.my_kernel ), my_factory( k.my_factory ) - {} - - const kernel_type my_kernel; - StreamFactory &my_factory; - }; - - template - class args_storage : public args_storage_base { - typedef typename args_storage_base::output_ports_type output_ports_type; - - // ---------- Update events helpers ---------- // - template - bool do_try_put( const kernel_input_tuple& ip, output_ports_type &op ) const { - const auto& t = get( ip ); - auto &port = get( op ); - return port.try_put( t ); - } - - template - bool do_try_put( const kernel_input_tuple& ip, output_ports_type &op, internal::sequence ) const { - return internal::or_return_values( do_try_put( ip, op )... ); - } - - // ------------------------------------------- // - class run_kernel_func : tbb::internal::no_assign { - public: - run_kernel_func( kernel_input_tuple &ip, const streaming_node &node, const args_storage& storage ) - : my_kernel_func( ip, node, storage, get<0>(ip).device() ) {} - - // It is immpossible to use Args... because a function pointer cannot be casted to a function reference implicitly. - // Allow the compiler to deduce types for function pointers automatically. - template - void operator()( FnArgs&... args ) { - internal::convert_and_call_impl::doit( my_kernel_func, my_kernel_func.my_ip, args... ); - } - private: - struct kernel_func : tbb::internal::no_copy { - kernel_input_tuple &my_ip; - const streaming_node &my_node; - const args_storage& my_storage; - device_type my_device; - - kernel_func( kernel_input_tuple &ip, const streaming_node &node, const args_storage& storage, device_type device ) - : my_ip( ip ), my_node( node ), my_storage( storage ), my_device( device ) - {} - - template - void operator()( FnArgs&... args ) { - my_node.enqueue_kernel( my_ip, my_storage.my_factory, my_device, my_storage.my_kernel, args... ); - } - } my_kernel_func; - }; - - template - class run_finalize_func : tbb::internal::no_assign { - public: - run_finalize_func( kernel_input_tuple &ip, StreamFactory &factory, FinalizeFn fn ) - : my_ip( ip ), my_finalize_func( factory, get<0>(ip).device(), fn ) {} - - // It is immpossible to use Args... because a function pointer cannot be casted to a function reference implicitly. - // Allow the compiler to deduce types for function pointers automatically. - template - void operator()( FnArgs&... args ) { - internal::convert_and_call_impl::doit( my_finalize_func, my_ip, args... ); - } - private: - kernel_input_tuple &my_ip; - - struct finalize_func : tbb::internal::no_assign { - StreamFactory &my_factory; - device_type my_device; - FinalizeFn my_fn; - - finalize_func( StreamFactory &factory, device_type device, FinalizeFn fn ) - : my_factory(factory), my_device(device), my_fn(fn) {} - - template - void operator()( FnArgs&... args ) { - my_factory.finalize( my_device, my_fn, args... ); - } - } my_finalize_func; - }; - - template - static run_finalize_func make_run_finalize_func( kernel_input_tuple &ip, StreamFactory &factory, FinalizeFn fn ) { - return run_finalize_func( ip, factory, fn ); - } - - class send_func : tbb::internal::no_assign { - public: - send_func( StreamFactory &factory, device_type d ) - : my_factory(factory), my_device( d ) {} - - template - void operator()( FnArgs&... args ) { - my_factory.send_data( my_device, args... ); - } - private: - StreamFactory &my_factory; - device_type my_device; - }; - - public: - args_storage( const kernel_type& kernel, StreamFactory &f, Args&&... args ) - : args_storage_base( kernel, f ) - , my_args_pack( std::forward(args)... ) - {} - - args_storage( const args_storage &k ) : args_storage_base( k ), my_args_pack( k.my_args_pack ) {} - - args_storage( const args_storage_base &k, Args&&... args ) : args_storage_base( k ), my_args_pack( std::forward(args)... ) {} - - void enqueue( kernel_input_tuple &ip, output_ports_type &op, const streaming_node &n ) __TBB_override { - // Make const qualified args_pack (from non-const) - const args_pack_type& const_args_pack = my_args_pack; - // factory.enqure_kernel() gets - // - 'ip' tuple elements by reference and updates it (and 'ip') with dependencies - // - arguments (from my_args_pack) by const-reference via const_args_pack - tbb::internal::call( run_kernel_func( ip, n, *this ), const_args_pack ); - - if (! do_try_put( ip, op, input_sequence() ) ) { - graph& g = n.my_graph; - // No one message was passed to successors so set a callback to extend the graph lifetime until the kernel completion. - g.increment_wait_count(); - - // factory.finalize() gets - // - 'ip' tuple elements by reference, so 'ip' might be changed - // - arguments (from my_args_pack) by const-reference via const_args_pack - tbb::internal::call( make_run_finalize_func(ip, this->my_factory, [&g] { - g.decrement_wait_count(); - }), const_args_pack ); - } - } - - void send( device_type d ) __TBB_override { - // factory.send() gets arguments by reference and updates these arguments with dependencies - // (it gets but usually ignores port_ref-s) - tbb::internal::call( send_func( this->my_factory, d ), my_args_pack ); - } - - args_storage_base *clone() const __TBB_override { - // Create new args_storage with copying constructor. - return new args_storage( *this ); - } - - private: - typedef tbb::internal::stored_pack args_pack_type; - args_pack_type my_args_pack; - }; - - // Body for kernel_multifunction_node. - class kernel_body : tbb::internal::no_assign { - public: - kernel_body( const streaming_node &node ) : my_node( node ) {} - - void operator()( kernel_input_tuple ip, typename args_storage_base::output_ports_type &op ) { - __TBB_ASSERT( (my_node.my_args_storage != NULL), "No arguments storage" ); - // 'ip' is passed by value to create local copy for updating inside enqueue_kernel() - my_node.my_args_storage->enqueue( ip, op, my_node ); - } - private: - const streaming_node &my_node; - }; - - template ::type > - struct wrap_to_async { - typedef T type; // Keep port_ref as it is - }; - - template - struct wrap_to_async { - typedef typename StreamFactory::template async_msg_type< typename tbb::internal::strip::type > type; - }; - - template - args_storage_base *make_args_storage(const args_storage_base& storage, Args&&... args) const { - // In this variadic template convert all simple types 'T' into 'async_msg_type' - return new args_storage(storage, std::forward(args)...); - } - - void notify_new_device( device_type d ) { - my_args_storage->send( d ); - } - - template - void enqueue_kernel( kernel_input_tuple& ip, StreamFactory& factory, device_type device, const kernel_type& kernel, Args&... args ) const { - this->enqueue_kernel_impl( ip, factory, device, kernel, args... ); - } - -public: - template - streaming_node( graph &g, const kernel_type& kernel, DeviceSelector d, StreamFactory &f ) - : base_type( g ) - , my_indexer_node( g ) - , my_device_selector( new device_selector( d, *this, f ) ) - , my_device_selector_node( g, serial, device_selector_body( my_device_selector ) ) - , my_join_node( g ) - , my_kernel_node( g, serial, kernel_body( *this ) ) - // By default, streaming_node maps all its ports to the kernel arguments on a one-to-one basis. - , my_args_storage( make_args_storage( args_storage<>(kernel, f), port_ref<0, NUM_INPUTS - 1>() ) ) - { - base_type::set_external_ports( get_input_ports(), get_output_ports() ); - make_edges(); - } - - streaming_node( const streaming_node &node ) - : base_type( node.my_graph ) - , my_indexer_node( node.my_indexer_node ) - , my_device_selector( node.my_device_selector->clone( *this ) ) - , my_device_selector_node( node.my_graph, serial, device_selector_body( my_device_selector ) ) - , my_join_node( node.my_join_node ) - , my_kernel_node( node.my_graph, serial, kernel_body( *this ) ) - , my_args_storage( node.my_args_storage->clone() ) - { - base_type::set_external_ports( get_input_ports(), get_output_ports() ); - make_edges(); - } - - streaming_node( streaming_node &&node ) - : base_type( node.my_graph ) - , my_indexer_node( std::move( node.my_indexer_node ) ) - , my_device_selector( node.my_device_selector->clone(*this) ) - , my_device_selector_node( node.my_graph, serial, device_selector_body( my_device_selector ) ) - , my_join_node( std::move( node.my_join_node ) ) - , my_kernel_node( node.my_graph, serial, kernel_body( *this ) ) - , my_args_storage( node.my_args_storage ) - { - base_type::set_external_ports( get_input_ports(), get_output_ports() ); - make_edges(); - // Set moving node mappers to NULL to prevent double deallocation. - node.my_args_storage = NULL; - } - - ~streaming_node() { - if ( my_args_storage ) delete my_args_storage; - if ( my_device_selector ) delete my_device_selector; - } - - template - void set_args( Args&&... args ) { - // Copy the base class of args_storage and create new storage for "Args...". - args_storage_base * const new_args_storage = make_args_storage( *my_args_storage, typename wrap_to_async::type(std::forward(args))...); - delete my_args_storage; - my_args_storage = new_args_storage; - } - -protected: - void reset_node( reset_flags = rf_reset_protocol ) __TBB_override { __TBB_ASSERT( false, "Not implemented yet" ); } - -private: - indexer_node_type my_indexer_node; - device_selector_base *my_device_selector; - device_selector_node my_device_selector_node; - join_node my_join_node; - kernel_multifunction_node my_kernel_node; - - args_storage_base *my_args_storage; -}; - -#endif // __TBB_PREVIEW_STREAMING_NODE -#endif // __TBB_flow_graph_streaming_H diff --git a/src/3rd party/tbb/internal/_flow_graph_tagged_buffer_impl.h b/src/3rd party/tbb/internal/_flow_graph_tagged_buffer_impl.h deleted file mode 100644 index 87a32fe604..0000000000 --- a/src/3rd party/tbb/internal/_flow_graph_tagged_buffer_impl.h +++ /dev/null @@ -1,249 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// a hash table buffer that can expand, and can support as many deletions as -// additions, list-based, with elements of list held in array (for destruction -// management), multiplicative hashing (like ets). No synchronization built-in. -// - -#ifndef __TBB__flow_graph_hash_buffer_impl_H -#define __TBB__flow_graph_hash_buffer_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -// included in namespace tbb::flow::interfaceX::internal - -// elements in the table are a simple list; we need pointer to next element to -// traverse the chain -template -struct buffer_element_type { - // the second parameter below is void * because we can't forward-declare the type - // itself, so we just reinterpret_cast below. - typedef typename aligned_pair::type type; -}; - -template - < - typename Key, // type of key within ValueType - typename ValueType, - typename ValueToKey, // abstract method that returns "const Key" or "const Key&" given ValueType - typename HashCompare, // has hash and equal - typename Allocator=tbb::cache_aligned_allocator< typename aligned_pair::type > - > -class hash_buffer : public HashCompare { -public: - static const size_t INITIAL_SIZE = 8; // initial size of the hash pointer table - typedef ValueType value_type; - typedef typename buffer_element_type< value_type >::type element_type; - typedef value_type *pointer_type; - typedef element_type *list_array_type; // array we manage manually - typedef list_array_type *pointer_array_type; - typedef typename Allocator::template rebind::other pointer_array_allocator_type; - typedef typename Allocator::template rebind::other elements_array_allocator; - typedef typename tbb::internal::strip::type Knoref; - -private: - ValueToKey *my_key; - size_t my_size; - size_t nelements; - pointer_array_type pointer_array; // pointer_array[my_size] - list_array_type elements_array; // elements_array[my_size / 2] - element_type* free_list; - - size_t mask() { return my_size - 1; } - - void set_up_free_list( element_type **p_free_list, list_array_type la, size_t sz) { - for(size_t i=0; i < sz - 1; ++i ) { // construct free list - la[i].second = &(la[i+1]); - } - la[sz-1].second = NULL; - *p_free_list = (element_type *)&(la[0]); - } - - // cleanup for exceptions - struct DoCleanup { - pointer_array_type *my_pa; - list_array_type *my_elements; - size_t my_size; - - DoCleanup(pointer_array_type &pa, list_array_type &my_els, size_t sz) : - my_pa(&pa), my_elements(&my_els), my_size(sz) { } - ~DoCleanup() { - if(my_pa) { - size_t dont_care = 0; - internal_free_buffer(*my_pa, *my_elements, my_size, dont_care); - } - } - }; - - // exception-safety requires we do all the potentially-throwing operations first - void grow_array() { - size_t new_size = my_size*2; - size_t new_nelements = nelements; // internal_free_buffer zeroes this - list_array_type new_elements_array = NULL; - pointer_array_type new_pointer_array = NULL; - list_array_type new_free_list = NULL; - { - DoCleanup my_cleanup(new_pointer_array, new_elements_array, new_size); - new_elements_array = elements_array_allocator().allocate(my_size); - new_pointer_array = pointer_array_allocator_type().allocate(new_size); - for(size_t i=0; i < new_size; ++i) new_pointer_array[i] = NULL; - set_up_free_list(&new_free_list, new_elements_array, my_size ); - - for(size_t i=0; i < my_size; ++i) { - for( element_type* op = pointer_array[i]; op; op = (element_type *)(op->second)) { - value_type *ov = reinterpret_cast(&(op->first)); - // could have std::move semantics - internal_insert_with_key(new_pointer_array, new_size, new_free_list, *ov); - } - } - my_cleanup.my_pa = NULL; - my_cleanup.my_elements = NULL; - } - - internal_free_buffer(pointer_array, elements_array, my_size, nelements); - free_list = new_free_list; - pointer_array = new_pointer_array; - elements_array = new_elements_array; - my_size = new_size; - nelements = new_nelements; - } - - // v should have perfect forwarding if std::move implemented. - // we use this method to move elements in grow_array, so can't use class fields - void internal_insert_with_key( element_type **p_pointer_array, size_t p_sz, list_array_type &p_free_list, - const value_type &v) { - size_t l_mask = p_sz-1; - __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); - size_t h = this->hash((*my_key)(v)) & l_mask; - __TBB_ASSERT(p_free_list, "Error: free list not set up."); - element_type* my_elem = p_free_list; p_free_list = (element_type *)(p_free_list->second); - (void) new(&(my_elem->first)) value_type(v); - my_elem->second = p_pointer_array[h]; - p_pointer_array[h] = my_elem; - } - - void internal_initialize_buffer() { - pointer_array = pointer_array_allocator_type().allocate(my_size); - for(size_t i = 0; i < my_size; ++i) pointer_array[i] = NULL; - elements_array = elements_array_allocator().allocate(my_size / 2); - set_up_free_list(&free_list, elements_array, my_size / 2); - } - - // made static so an enclosed class can use to properly dispose of the internals - static void internal_free_buffer( pointer_array_type &pa, list_array_type &el, size_t &sz, size_t &ne ) { - if(pa) { - for(size_t i = 0; i < sz; ++i ) { - element_type *p_next; - for( element_type *p = pa[i]; p; p = p_next) { - p_next = (element_type *)p->second; - internal::punned_cast(&(p->first))->~value_type(); - } - } - pointer_array_allocator_type().deallocate(pa, sz); - pa = NULL; - } - // Separate test (if allocation of pa throws, el may be allocated. - // but no elements will be constructed.) - if(el) { - elements_array_allocator().deallocate(el, sz / 2); - el = NULL; - } - sz = INITIAL_SIZE; - ne = 0; - } - -public: - hash_buffer() : my_key(NULL), my_size(INITIAL_SIZE), nelements(0) { - internal_initialize_buffer(); - } - - ~hash_buffer() { - internal_free_buffer(pointer_array, elements_array, my_size, nelements); - if(my_key) delete my_key; - } - - void reset() { - internal_free_buffer(pointer_array, elements_array, my_size, nelements); - internal_initialize_buffer(); - } - - // Take ownership of func object allocated with new. - // This method is only used internally, so can't be misused by user. - void set_key_func(ValueToKey *vtk) { my_key = vtk; } - // pointer is used to clone() - ValueToKey* get_key_func() { return my_key; } - - bool insert_with_key(const value_type &v) { - pointer_type p = NULL; - __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); - if(find_ref_with_key((*my_key)(v), p)) { - p->~value_type(); - (void) new(p) value_type(v); // copy-construct into the space - return false; - } - ++nelements; - if(nelements*2 > my_size) grow_array(); - internal_insert_with_key(pointer_array, my_size, free_list, v); - return true; - } - - // returns true and sets v to array element if found, else returns false. - bool find_ref_with_key(const Knoref& k, pointer_type &v) { - size_t i = this->hash(k) & mask(); - for(element_type* p = pointer_array[i]; p; p = (element_type *)(p->second)) { - pointer_type pv = reinterpret_cast(&(p->first)); - __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); - if(this->equal((*my_key)(*pv), k)) { - v = pv; - return true; - } - } - return false; - } - - bool find_with_key( const Knoref& k, value_type &v) { - value_type *p; - if(find_ref_with_key(k, p)) { - v = *p; - return true; - } - else - return false; - } - - void delete_with_key(const Knoref& k) { - size_t h = this->hash(k) & mask(); - element_type* prev = NULL; - for(element_type* p = pointer_array[h]; p; prev = p, p = (element_type *)(p->second)) { - value_type *vp = reinterpret_cast(&(p->first)); - __TBB_ASSERT(my_key, "Error: value-to-key functor not provided"); - if(this->equal((*my_key)(*vp), k)) { - vp->~value_type(); - if(prev) prev->second = p->second; - else pointer_array[h] = (element_type *)(p->second); - p->second = free_list; - free_list = p; - --nelements; - return; - } - } - __TBB_ASSERT(false, "key not found for delete"); - } -}; -#endif // __TBB__flow_graph_hash_buffer_impl_H diff --git a/src/3rd party/tbb/internal/_flow_graph_trace_impl.h b/src/3rd party/tbb/internal/_flow_graph_trace_impl.h deleted file mode 100644 index 6299cde517..0000000000 --- a/src/3rd party/tbb/internal/_flow_graph_trace_impl.h +++ /dev/null @@ -1,364 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef _FGT_GRAPH_TRACE_IMPL_H -#define _FGT_GRAPH_TRACE_IMPL_H - -#include "../tbb_profiling.h" -#if (_MSC_VER >= 1900) - #include -#endif - -namespace tbb { - namespace internal { - -#if TBB_USE_THREADING_TOOLS - #if TBB_PREVIEW_FLOW_GRAPH_TRACE - #if (_MSC_VER >= 1900) - #define CODEPTR() (_ReturnAddress()) - #elif __TBB_GCC_VERSION >= 40800 - #define CODEPTR() ( __builtin_return_address(0)) - #else - #define CODEPTR() NULL - #endif - #else - #define CODEPTR() NULL - #endif /* TBB_PREVIEW_FLOW_GRAPH_TRACE */ - -static inline void fgt_alias_port(void *node, void *p, bool visible) { - if(visible) - itt_relation_add( ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_NODE ); - else - itt_relation_add( ITT_DOMAIN_FLOW, p, FLOW_NODE, __itt_relation_is_child_of, node, FLOW_NODE ); -} - -static inline void fgt_composite ( void* codeptr, void *node, void *graph ) { - itt_make_task_group( ITT_DOMAIN_FLOW, node, FLOW_NODE, graph, FLOW_GRAPH, FLOW_COMPOSITE_NODE ); - suppress_unused_warning( codeptr ); -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - if (codeptr != NULL) { - register_node_addr(ITT_DOMAIN_FLOW, node, FLOW_NODE, CODE_ADDRESS, &codeptr); - } -#endif -} - -static inline void fgt_internal_alias_input_port( void *node, void *p, string_index name_index ) { - itt_make_task_group( ITT_DOMAIN_FLOW, p, FLOW_INPUT_PORT, node, FLOW_NODE, name_index ); - itt_relation_add( ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_INPUT_PORT ); -} - -static inline void fgt_internal_alias_output_port( void *node, void *p, string_index name_index ) { - itt_make_task_group( ITT_DOMAIN_FLOW, p, FLOW_OUTPUT_PORT, node, FLOW_NODE, name_index ); - itt_relation_add( ITT_DOMAIN_FLOW, node, FLOW_NODE, __itt_relation_is_parent_of, p, FLOW_OUTPUT_PORT ); -} - -template -void alias_input_port(void *node, tbb::flow::receiver* port, string_index name_index) { - // TODO: Make fgt_internal_alias_input_port a function template? - fgt_internal_alias_input_port( node, port, name_index); -} - -template < typename PortsTuple, int N > -struct fgt_internal_input_alias_helper { - static void alias_port( void *node, PortsTuple &ports ) { - alias_input_port( node, &(tbb::flow::get(ports)), static_cast(FLOW_INPUT_PORT_0 + N - 1) ); - fgt_internal_input_alias_helper::alias_port( node, ports ); - } -}; - -template < typename PortsTuple > -struct fgt_internal_input_alias_helper { - static void alias_port( void * /* node */, PortsTuple & /* ports */ ) { } -}; - -template -void alias_output_port(void *node, tbb::flow::sender* port, string_index name_index) { - // TODO: Make fgt_internal_alias_output_port a function template? - fgt_internal_alias_output_port( node, static_cast(port), name_index); -} - -template < typename PortsTuple, int N > -struct fgt_internal_output_alias_helper { - static void alias_port( void *node, PortsTuple &ports ) { - alias_output_port( node, &(tbb::flow::get(ports)), static_cast(FLOW_OUTPUT_PORT_0 + N - 1) ); - fgt_internal_output_alias_helper::alias_port( node, ports ); - } -}; - -template < typename PortsTuple > -struct fgt_internal_output_alias_helper { - static void alias_port( void * /*node*/, PortsTuple &/*ports*/ ) { - } -}; - -static inline void fgt_internal_create_input_port( void *node, void *p, string_index name_index ) { - itt_make_task_group( ITT_DOMAIN_FLOW, p, FLOW_INPUT_PORT, node, FLOW_NODE, name_index ); -} - -static inline void fgt_internal_create_output_port( void* codeptr, void *node, void *p, string_index name_index ) { - itt_make_task_group(ITT_DOMAIN_FLOW, p, FLOW_OUTPUT_PORT, node, FLOW_NODE, name_index); - suppress_unused_warning( codeptr ); -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - if (codeptr != NULL) { - register_node_addr(ITT_DOMAIN_FLOW, node, FLOW_NODE, CODE_ADDRESS, &codeptr); - } -#endif -} - -template -void register_input_port(void *node, tbb::flow::receiver* port, string_index name_index) { - // TODO: Make fgt_internal_create_input_port a function template? - // In C++03 dependent name lookup from the template definition context - // works only for function declarations with external linkage: - // http://www.open-std.org/JTC1/SC22/WG21/docs/cwg_defects.html#561 - fgt_internal_create_input_port(node, static_cast(port), name_index); -} - -template < typename PortsTuple, int N > -struct fgt_internal_input_helper { - static void register_port( void *node, PortsTuple &ports ) { - register_input_port( node, &(tbb::flow::get(ports)), static_cast(FLOW_INPUT_PORT_0 + N - 1) ); - fgt_internal_input_helper::register_port( node, ports ); - } -}; - -template < typename PortsTuple > -struct fgt_internal_input_helper { - static void register_port( void *node, PortsTuple &ports ) { - register_input_port( node, &(tbb::flow::get<0>(ports)), FLOW_INPUT_PORT_0 ); - } -}; - -template -void register_output_port(void* codeptr, void *node, tbb::flow::sender* port, string_index name_index) { - // TODO: Make fgt_internal_create_output_port a function template? - fgt_internal_create_output_port( codeptr, node, static_cast(port), name_index); -} - -template < typename PortsTuple, int N > -struct fgt_internal_output_helper { - static void register_port( void* codeptr, void *node, PortsTuple &ports ) { - register_output_port( codeptr, node, &(tbb::flow::get(ports)), static_cast(FLOW_OUTPUT_PORT_0 + N - 1) ); - fgt_internal_output_helper::register_port( codeptr, node, ports ); - } -}; - -template < typename PortsTuple > -struct fgt_internal_output_helper { - static void register_port( void* codeptr, void *node, PortsTuple &ports ) { - register_output_port( codeptr, node, &(tbb::flow::get<0>(ports)), FLOW_OUTPUT_PORT_0 ); - } -}; - -template< typename NodeType > -void fgt_multioutput_node_desc( const NodeType *node, const char *desc ) { - void *addr = (void *)( static_cast< tbb::flow::receiver< typename NodeType::input_type > * >(const_cast< NodeType *>(node)) ); - itt_metadata_str_add( ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc ); -} - -template< typename NodeType > -void fgt_multiinput_multioutput_node_desc( const NodeType *node, const char *desc ) { - void *addr = const_cast(node); - itt_metadata_str_add( ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc ); -} - -template< typename NodeType > -static inline void fgt_node_desc( const NodeType *node, const char *desc ) { - void *addr = (void *)( static_cast< tbb::flow::sender< typename NodeType::output_type > * >(const_cast< NodeType *>(node)) ); - itt_metadata_str_add( ITT_DOMAIN_FLOW, addr, FLOW_NODE, FLOW_OBJECT_NAME, desc ); -} - -static inline void fgt_graph_desc( void *g, const char *desc ) { - itt_metadata_str_add( ITT_DOMAIN_FLOW, g, FLOW_GRAPH, FLOW_OBJECT_NAME, desc ); -} - -static inline void fgt_body( void *node, void *body ) { - itt_relation_add( ITT_DOMAIN_FLOW, body, FLOW_BODY, __itt_relation_is_child_of, node, FLOW_NODE ); -} - -template< int N, typename PortsTuple > -static inline void fgt_multioutput_node(void* codeptr, string_index t, void *g, void *input_port, PortsTuple &ports ) { - itt_make_task_group( ITT_DOMAIN_FLOW, input_port, FLOW_NODE, g, FLOW_GRAPH, t ); - fgt_internal_create_input_port( input_port, input_port, FLOW_INPUT_PORT_0 ); - fgt_internal_output_helper::register_port(codeptr, input_port, ports ); -} - -template< int N, typename PortsTuple > -static inline void fgt_multioutput_node_with_body( void* codeptr, string_index t, void *g, void *input_port, PortsTuple &ports, void *body ) { - itt_make_task_group( ITT_DOMAIN_FLOW, input_port, FLOW_NODE, g, FLOW_GRAPH, t ); - fgt_internal_create_input_port( input_port, input_port, FLOW_INPUT_PORT_0 ); - fgt_internal_output_helper::register_port( codeptr, input_port, ports ); - fgt_body( input_port, body ); -} - -template< int N, typename PortsTuple > -static inline void fgt_multiinput_node( void* codeptr, string_index t, void *g, PortsTuple &ports, void *output_port) { - itt_make_task_group( ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t ); - fgt_internal_create_output_port( codeptr, output_port, output_port, FLOW_OUTPUT_PORT_0 ); - fgt_internal_input_helper::register_port( output_port, ports ); -} - -static inline void fgt_multiinput_multioutput_node( void* codeptr, string_index t, void *n, void *g ) { - itt_make_task_group( ITT_DOMAIN_FLOW, n, FLOW_NODE, g, FLOW_GRAPH, t ); - suppress_unused_warning( codeptr ); -#if TBB_PREVIEW_FLOW_GRAPH_TRACE - if (codeptr != NULL) { - register_node_addr(ITT_DOMAIN_FLOW, n, FLOW_NODE, CODE_ADDRESS, &codeptr); - } -#endif -} - -static inline void fgt_node( void* codeptr, string_index t, void *g, void *output_port ) { - itt_make_task_group( ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t ); - fgt_internal_create_output_port( codeptr, output_port, output_port, FLOW_OUTPUT_PORT_0 ); -} - -static void fgt_node_with_body( void* codeptr, string_index t, void *g, void *output_port, void *body ) { - itt_make_task_group( ITT_DOMAIN_FLOW, output_port, FLOW_NODE, g, FLOW_GRAPH, t ); - fgt_internal_create_output_port(codeptr, output_port, output_port, FLOW_OUTPUT_PORT_0 ); - fgt_body( output_port, body ); -} - -static inline void fgt_node( void* codeptr, string_index t, void *g, void *input_port, void *output_port ) { - fgt_node( codeptr, t, g, output_port ); - fgt_internal_create_input_port( output_port, input_port, FLOW_INPUT_PORT_0 ); -} - -static inline void fgt_node_with_body( void* codeptr, string_index t, void *g, void *input_port, void *output_port, void *body ) { - fgt_node_with_body( codeptr, t, g, output_port, body ); - fgt_internal_create_input_port( output_port, input_port, FLOW_INPUT_PORT_0 ); -} - - -static inline void fgt_node( void* codeptr, string_index t, void *g, void *input_port, void *decrement_port, void *output_port ) { - fgt_node( codeptr, t, g, input_port, output_port ); - fgt_internal_create_input_port( output_port, decrement_port, FLOW_INPUT_PORT_1 ); -} - -static inline void fgt_make_edge( void *output_port, void *input_port ) { - itt_relation_add( ITT_DOMAIN_FLOW, output_port, FLOW_OUTPUT_PORT, __itt_relation_is_predecessor_to, input_port, FLOW_INPUT_PORT); -} - -static inline void fgt_remove_edge( void *output_port, void *input_port ) { - itt_relation_add( ITT_DOMAIN_FLOW, output_port, FLOW_OUTPUT_PORT, __itt_relation_is_sibling_of, input_port, FLOW_INPUT_PORT); -} - -static inline void fgt_graph( void *g ) { - itt_make_task_group( ITT_DOMAIN_FLOW, g, FLOW_GRAPH, NULL, FLOW_NULL, FLOW_GRAPH ); -} - -static inline void fgt_begin_body( void *body ) { - itt_task_begin( ITT_DOMAIN_FLOW, body, FLOW_BODY, NULL, FLOW_NULL, FLOW_BODY ); -} - -static inline void fgt_end_body( void * ) { - itt_task_end( ITT_DOMAIN_FLOW ); -} - -static inline void fgt_async_try_put_begin( void *node, void *port ) { - itt_task_begin( ITT_DOMAIN_FLOW, port, FLOW_OUTPUT_PORT, node, FLOW_NODE, FLOW_OUTPUT_PORT ); -} - -static inline void fgt_async_try_put_end( void *, void * ) { - itt_task_end( ITT_DOMAIN_FLOW ); -} - -static inline void fgt_async_reserve( void *node, void *graph ) { - itt_region_begin( ITT_DOMAIN_FLOW, node, FLOW_NODE, graph, FLOW_GRAPH, FLOW_NULL ); -} - -static inline void fgt_async_commit( void *node, void * /*graph*/) { - itt_region_end( ITT_DOMAIN_FLOW, node, FLOW_NODE ); -} - -static inline void fgt_reserve_wait( void *graph ) { - itt_region_begin( ITT_DOMAIN_FLOW, graph, FLOW_GRAPH, NULL, FLOW_NULL, FLOW_NULL ); -} - -static inline void fgt_release_wait( void *graph ) { - itt_region_end( ITT_DOMAIN_FLOW, graph, FLOW_GRAPH ); -} - -#else // TBB_USE_THREADING_TOOLS - -#define CODEPTR() NULL - -static inline void fgt_alias_port(void * /*node*/, void * /*p*/, bool /*visible*/ ) { } - -static inline void fgt_composite ( void* /*codeptr*/, void * /*node*/, void * /*graph*/ ) { } - -static inline void fgt_graph( void * /*g*/ ) { } - -template< typename NodeType > -static inline void fgt_multioutput_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { } - -template< typename NodeType > -static inline void fgt_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { } - -static inline void fgt_graph_desc( void * /*g*/, const char * /*desc*/ ) { } - -static inline void fgt_body( void * /*node*/, void * /*body*/ ) { } - -template< int N, typename PortsTuple > -static inline void fgt_multioutput_node( void* /*codeptr*/, string_index /*t*/, void * /*g*/, void * /*input_port*/, PortsTuple & /*ports*/ ) { } - -template< int N, typename PortsTuple > -static inline void fgt_multioutput_node_with_body( void* /*codeptr*/, string_index /*t*/, void * /*g*/, void * /*input_port*/, PortsTuple & /*ports*/, void * /*body*/ ) { } - -template< int N, typename PortsTuple > -static inline void fgt_multiinput_node( void* /*codeptr*/, string_index /*t*/, void * /*g*/, PortsTuple & /*ports*/, void * /*output_port*/ ) { } - -static inline void fgt_multiinput_multioutput_node( void* /*codeptr*/, string_index /*t*/, void * /*node*/, void * /*graph*/ ) { } - -static inline void fgt_node( void* /*codeptr*/, string_index /*t*/, void * /*g*/, void * /*output_port*/ ) { } -static inline void fgt_node( void* /*codeptr*/, string_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*output_port*/ ) { } -static inline void fgt_node( void* /*codeptr*/, string_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*decrement_port*/, void * /*output_port*/ ) { } - -static inline void fgt_node_with_body( void* /*codeptr*/, string_index /*t*/, void * /*g*/, void * /*output_port*/, void * /*body*/ ) { } -static inline void fgt_node_with_body( void* /*codeptr*/, string_index /*t*/, void * /*g*/, void * /*input_port*/, void * /*output_port*/, void * /*body*/ ) { } - -static inline void fgt_make_edge( void * /*output_port*/, void * /*input_port*/ ) { } -static inline void fgt_remove_edge( void * /*output_port*/, void * /*input_port*/ ) { } - -static inline void fgt_begin_body( void * /*body*/ ) { } -static inline void fgt_end_body( void * /*body*/) { } - -static inline void fgt_async_try_put_begin( void * /*node*/, void * /*port*/ ) { } -static inline void fgt_async_try_put_end( void * /*node*/ , void * /*port*/ ) { } -static inline void fgt_async_reserve( void * /*node*/, void * /*graph*/ ) { } -static inline void fgt_async_commit( void * /*node*/, void * /*graph*/ ) { } -static inline void fgt_reserve_wait( void * /*graph*/ ) { } -static inline void fgt_release_wait( void * /*graph*/ ) { } - -template< typename NodeType > -void fgt_multiinput_multioutput_node_desc( const NodeType * /*node*/, const char * /*desc*/ ) { } - -template < typename PortsTuple, int N > -struct fgt_internal_input_alias_helper { - static void alias_port( void * /*node*/, PortsTuple & /*ports*/ ) { } -}; - -template < typename PortsTuple, int N > -struct fgt_internal_output_alias_helper { - static void alias_port( void * /*node*/, PortsTuple & /*ports*/ ) { } -}; - -#endif // TBB_USE_THREADING_TOOLS - - } // namespace internal -} // namespace tbb - -#endif diff --git a/src/3rd party/tbb/internal/_flow_graph_types_impl.h b/src/3rd party/tbb/internal/_flow_graph_types_impl.h deleted file mode 100644 index e223dae6cb..0000000000 --- a/src/3rd party/tbb/internal/_flow_graph_types_impl.h +++ /dev/null @@ -1,723 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB__flow_graph_types_impl_H -#define __TBB__flow_graph_types_impl_H - -#ifndef __TBB_flow_graph_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -// included in namespace tbb::flow::interfaceX - -namespace internal { - - // the change to key_matching (adding a K and KHash template parameter, making it a class) - // means we have to pass this data to the key_matching_port. All the ports have only one - // template parameter, so we have to wrap the following types in a trait: - // - // . K == key_type - // . KHash == hash and compare for Key - // . TtoK == function_body that given an object of T, returns its K - // . T == type accepted by port, and stored in the hash table - // - // The port will have an additional parameter on node construction, which is a function_body - // that accepts a const T& and returns a K which is the field in T which is its K. - template - struct KeyTrait { - typedef Kp K; - typedef Tp T; - typedef internal::type_to_key_function_body TtoK; - typedef KHashp KHash; - }; - - // wrap each element of a tuple in a template, and make a tuple of the result. - template class PT, typename TypeTuple> - struct wrap_tuple_elements; - - // A wrapper that generates the traits needed for each port of a key-matching join, - // and the type of the tuple of input ports. - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements; - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_VARIADIC_TUPLE_PRESENT - template class PT, typename... Args> - struct wrap_tuple_elements >{ - typedef typename tbb::flow::tuple... > type; - }; - - template class PT, typename KeyTraits, typename... Args> - struct wrap_key_tuple_elements > { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef typename tbb::flow::tuple >... > type; - }; -#else - template class PT, typename TypeTuple> - struct wrap_tuple_elements<1, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<1, PT, KeyTraits, TypeTuple > { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef typename tbb::flow::tuple< PT > type; - }; - - template class PT, typename TypeTuple> - struct wrap_tuple_elements<2, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<2, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef typename tbb::flow::tuple< PT, PT > type; - }; - - template class PT, typename TypeTuple> - struct wrap_tuple_elements<3, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<3, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef typename tbb::flow::tuple< PT, PT, PT > type; - }; - - template class PT, typename TypeTuple> - struct wrap_tuple_elements<4, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<4, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef typename tbb::flow::tuple< PT, PT, PT, - PT > type; - }; - - template class PT, typename TypeTuple> - struct wrap_tuple_elements<5, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<5, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef typename tbb::flow::tuple< PT, PT, PT, - PT, PT > type; - }; - -#if __TBB_VARIADIC_MAX >= 6 - template class PT, typename TypeTuple> - struct wrap_tuple_elements<6, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<6, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef KeyTrait::type> KeyTrait5; - typedef typename tbb::flow::tuple< PT, PT, PT, PT, - PT, PT > type; - }; -#endif - -#if __TBB_VARIADIC_MAX >= 7 - template class PT, typename TypeTuple> - struct wrap_tuple_elements<7, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<7, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef KeyTrait::type> KeyTrait5; - typedef KeyTrait::type> KeyTrait6; - typedef typename tbb::flow::tuple< PT, PT, PT, PT, - PT, PT, PT > type; - }; -#endif - -#if __TBB_VARIADIC_MAX >= 8 - template class PT, typename TypeTuple> - struct wrap_tuple_elements<8, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<8, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef KeyTrait::type> KeyTrait5; - typedef KeyTrait::type> KeyTrait6; - typedef KeyTrait::type> KeyTrait7; - typedef typename tbb::flow::tuple< PT, PT, PT, PT, - PT, PT, PT, PT > type; - }; -#endif - -#if __TBB_VARIADIC_MAX >= 9 - template class PT, typename TypeTuple> - struct wrap_tuple_elements<9, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<9, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef KeyTrait::type> KeyTrait5; - typedef KeyTrait::type> KeyTrait6; - typedef KeyTrait::type> KeyTrait7; - typedef KeyTrait::type> KeyTrait8; - typedef typename tbb::flow::tuple< PT, PT, PT, PT, - PT, PT, PT, PT, PT > type; - }; -#endif - -#if __TBB_VARIADIC_MAX >= 10 - template class PT, typename TypeTuple> - struct wrap_tuple_elements<10, PT, TypeTuple> { - typedef typename tbb::flow::tuple< - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type>, - PT::type> > - type; - }; - - template class PT, typename KeyTraits, typename TypeTuple> - struct wrap_key_tuple_elements<10, PT, KeyTraits, TypeTuple> { - typedef typename KeyTraits::key_type K; - typedef typename KeyTraits::hash_compare_type KHash; - typedef KeyTrait::type> KeyTrait0; - typedef KeyTrait::type> KeyTrait1; - typedef KeyTrait::type> KeyTrait2; - typedef KeyTrait::type> KeyTrait3; - typedef KeyTrait::type> KeyTrait4; - typedef KeyTrait::type> KeyTrait5; - typedef KeyTrait::type> KeyTrait6; - typedef KeyTrait::type> KeyTrait7; - typedef KeyTrait::type> KeyTrait8; - typedef KeyTrait::type> KeyTrait9; - typedef typename tbb::flow::tuple< PT, PT, PT, PT, - PT, PT, PT, PT, PT, - PT > type; - }; -#endif -#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_VARIADIC_TUPLE_PRESENT */ - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template< int... S > class sequence {}; - - template< int N, int... S > - struct make_sequence : make_sequence < N - 1, N - 1, S... > {}; - - template< int... S > - struct make_sequence < 0, S... > { - typedef sequence type; - }; -#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ - -#if __TBB_INITIALIZER_LISTS_PRESENT - // Until C++14 std::initializer_list does not guarantee life time of contained objects. - template - class initializer_list_wrapper { - public: - typedef T value_type; - typedef const T& reference; - typedef const T& const_reference; - typedef size_t size_type; - - typedef T* iterator; - typedef const T* const_iterator; - - initializer_list_wrapper( std::initializer_list il ) __TBB_NOEXCEPT( true ) : my_begin( static_cast(malloc( il.size()*sizeof( T ) )) ) { - iterator dst = my_begin; - for ( typename std::initializer_list::const_iterator src = il.begin(); src != il.end(); ++src ) - new (dst++) T( *src ); - my_end = dst; - } - - initializer_list_wrapper( const initializer_list_wrapper& ilw ) __TBB_NOEXCEPT( true ) : my_begin( static_cast(malloc( ilw.size()*sizeof( T ) )) ) { - iterator dst = my_begin; - for ( typename std::initializer_list::const_iterator src = ilw.begin(); src != ilw.end(); ++src ) - new (dst++) T( *src ); - my_end = dst; - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - initializer_list_wrapper( initializer_list_wrapper&& ilw ) __TBB_NOEXCEPT( true ) : my_begin( ilw.my_begin ), my_end( ilw.my_end ) { - ilw.my_begin = ilw.my_end = NULL; - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - - ~initializer_list_wrapper() { - if ( my_begin ) - free( my_begin ); - } - - const_iterator begin() const __TBB_NOEXCEPT(true) { return my_begin; } - const_iterator end() const __TBB_NOEXCEPT(true) { return my_end; } - size_t size() const __TBB_NOEXCEPT(true) { return (size_t)(my_end - my_begin); } - - private: - iterator my_begin; - iterator my_end; - }; -#endif /* __TBB_INITIALIZER_LISTS_PRESENT */ - -//! type mimicking std::pair but with trailing fill to ensure each element of an array -//* will have the correct alignment - template - struct type_plus_align { - char first[sizeof(T1)]; - T2 second; - char fill1[REM]; - }; - - template - struct type_plus_align { - char first[sizeof(T1)]; - T2 second; - }; - - template struct alignment_of { - typedef struct { char t; U padded; } test_alignment; - static const size_t value = sizeof(test_alignment) - sizeof(U); - }; - - // T1, T2 are actual types stored. The space defined for T1 in the type returned - // is a char array of the correct size. Type T2 should be trivially-constructible, - // T1 must be explicitly managed. - template - struct aligned_pair { - static const size_t t1_align = alignment_of::value; - static const size_t t2_align = alignment_of::value; - typedef type_plus_align just_pair; - static const size_t max_align = t1_align < t2_align ? t2_align : t1_align; - static const size_t extra_bytes = sizeof(just_pair) % max_align; - static const size_t remainder = extra_bytes ? max_align - extra_bytes : 0; - public: - typedef type_plus_align type; - }; // aligned_pair - -// support for variant type -// type we use when we're not storing a value -struct default_constructed { }; - -// type which contains another type, tests for what type is contained, and references to it. -// internal::Wrapper -// void CopyTo( void *newSpace) : builds a Wrapper copy of itself in newSpace - -// struct to allow us to copy and test the type of objects -struct WrapperBase { - virtual ~WrapperBase() {} - virtual void CopyTo(void* /*newSpace*/) const { } -}; - -// Wrapper contains a T, with the ability to test what T is. The Wrapper can be -// constructed from a T, can be copy-constructed from another Wrapper, and can be -// examined via value(), but not modified. -template -struct Wrapper: public WrapperBase { - typedef T value_type; - typedef T* pointer_type; -private: - T value_space; -public: - const value_type &value() const { return value_space; } - -private: - Wrapper(); - - // on exception will ensure the Wrapper will contain only a trivially-constructed object - struct _unwind_space { - pointer_type space; - _unwind_space(pointer_type p) : space(p) {} - ~_unwind_space() { - if(space) (void) new (space) Wrapper(default_constructed()); - } - }; -public: - explicit Wrapper( const T& other ) : value_space(other) { } - explicit Wrapper(const Wrapper& other) : value_space(other.value_space) { } - - void CopyTo(void* newSpace) const __TBB_override { - _unwind_space guard((pointer_type)newSpace); - (void) new(newSpace) Wrapper(value_space); - guard.space = NULL; - } - ~Wrapper() { } -}; - -// specialization for array objects -template -struct Wrapper : public WrapperBase { - typedef T value_type; - typedef T* pointer_type; - // space must be untyped. - typedef T ArrayType[N]; -private: - // The space is not of type T[N] because when copy-constructing, it would be - // default-initialized and then copied to in some fashion, resulting in two - // constructions and one destruction per element. If the type is char[ ], we - // placement new into each element, resulting in one construction per element. - static const size_t space_size = sizeof(ArrayType) / sizeof(char); - char value_space[space_size]; - - - // on exception will ensure the already-built objects will be destructed - // (the value_space is a char array, so it is already trivially-destructible.) - struct _unwind_class { - pointer_type space; - int already_built; - _unwind_class(pointer_type p) : space(p), already_built(0) {} - ~_unwind_class() { - if(space) { - for(size_t i = already_built; i > 0 ; --i ) space[i-1].~value_type(); - (void) new(space) Wrapper(default_constructed()); - } - } - }; -public: - const ArrayType &value() const { - char *vp = const_cast(value_space); - return reinterpret_cast(*vp); - } - -private: - Wrapper(); -public: - // have to explicitly construct because other decays to a const value_type* - explicit Wrapper(const ArrayType& other) { - _unwind_class guard((pointer_type)value_space); - pointer_type vp = reinterpret_cast(&value_space); - for(size_t i = 0; i < N; ++i ) { - (void) new(vp++) value_type(other[i]); - ++(guard.already_built); - } - guard.space = NULL; - } - explicit Wrapper(const Wrapper& other) : WrapperBase() { - // we have to do the heavy lifting to copy contents - _unwind_class guard((pointer_type)value_space); - pointer_type dp = reinterpret_cast(value_space); - pointer_type sp = reinterpret_cast(const_cast(other.value_space)); - for(size_t i = 0; i < N; ++i, ++dp, ++sp) { - (void) new(dp) value_type(*sp); - ++(guard.already_built); - } - guard.space = NULL; - } - - void CopyTo(void* newSpace) const __TBB_override { - (void) new(newSpace) Wrapper(*this); // exceptions handled in copy constructor - } - - ~Wrapper() { - // have to destroy explicitly in reverse order - pointer_type vp = reinterpret_cast(&value_space); - for(size_t i = N; i > 0 ; --i ) vp[i-1].~value_type(); - } -}; - -// given a tuple, return the type of the element that has the maximum alignment requirement. -// Given a tuple and that type, return the number of elements of the object with the max -// alignment requirement that is at least as big as the largest object in the tuple. - -template struct pick_one; -template struct pick_one { typedef T1 type; }; -template struct pick_one { typedef T2 type; }; - -template< template class Selector, typename T1, typename T2 > -struct pick_max { - typedef typename pick_one< (Selector::value > Selector::value), T1, T2 >::type type; -}; - -template struct size_of { static const int value = sizeof(T); }; - -template< size_t N, class Tuple, template class Selector > struct pick_tuple_max { - typedef typename pick_tuple_max::type LeftMaxType; - typedef typename tbb::flow::tuple_element::type ThisType; - typedef typename pick_max::type type; -}; - -template< class Tuple, template class Selector > struct pick_tuple_max<0, Tuple, Selector> { - typedef typename tbb::flow::tuple_element<0, Tuple>::type type; -}; - -// is the specified type included in a tuple? -template -struct is_element_of { - typedef typename tbb::flow::tuple_element::type T_i; - static const bool value = tbb::internal::is_same_type::value || is_element_of::value; -}; - -template -struct is_element_of { - typedef typename tbb::flow::tuple_element<0, Tuple>::type T_i; - static const bool value = tbb::internal::is_same_type::value; -}; - -// allow the construction of types that are listed tuple. If a disallowed type -// construction is written, a method involving this type is created. The -// type has no definition, so a syntax error is generated. -template struct ERROR_Type_Not_allowed_In_Tagged_Msg_Not_Member_Of_Tuple; - -template struct do_if; -template -struct do_if { - static void construct(void *mySpace, const T& x) { - (void) new(mySpace) Wrapper(x); - } -}; -template -struct do_if { - static void construct(void * /*mySpace*/, const T& x) { - // This method is instantiated when the type T does not match any of the - // element types in the Tuple in variant. - ERROR_Type_Not_allowed_In_Tagged_Msg_Not_Member_Of_Tuple::bad_type(x); - } -}; - -// Tuple tells us the allowed types that variant can hold. It determines the alignment of the space in -// Wrapper, and how big Wrapper is. -// -// the object can only be tested for type, and a read-only reference can be fetched by cast_to(). - -using tbb::internal::punned_cast; -struct tagged_null_type {}; -template -class tagged_msg { - typedef tbb::flow::tuple= 6 - , T5 - #endif - #if __TBB_VARIADIC_MAX >= 7 - , T6 - #endif - #if __TBB_VARIADIC_MAX >= 8 - , T7 - #endif - #if __TBB_VARIADIC_MAX >= 9 - , T8 - #endif - #if __TBB_VARIADIC_MAX >= 10 - , T9 - #endif - > Tuple; - -private: - class variant { - static const size_t N = tbb::flow::tuple_size::value; - typedef typename pick_tuple_max::type AlignType; - typedef typename pick_tuple_max::type MaxSizeType; - static const size_t MaxNBytes = (sizeof(Wrapper)+sizeof(AlignType)-1); - static const size_t MaxNElements = MaxNBytes/sizeof(AlignType); - typedef typename tbb::aligned_space SpaceType; - SpaceType my_space; - static const size_t MaxSize = sizeof(SpaceType); - - public: - variant() { (void) new(&my_space) Wrapper(default_constructed()); } - - template - variant( const T& x ) { - do_if::value>::construct(&my_space,x); - } - - variant(const variant& other) { - const WrapperBase * h = punned_cast(&(other.my_space)); - h->CopyTo(&my_space); - } - - // assignment must destroy and re-create the Wrapper type, as there is no way - // to create a Wrapper-to-Wrapper assign even if we find they agree in type. - void operator=( const variant& rhs ) { - if(&rhs != this) { - WrapperBase *h = punned_cast(&my_space); - h->~WrapperBase(); - const WrapperBase *ch = punned_cast(&(rhs.my_space)); - ch->CopyTo(&my_space); - } - } - - template - const U& variant_cast_to() const { - const Wrapper *h = dynamic_cast*>(punned_cast(&my_space)); - if(!h) { - tbb::internal::throw_exception(tbb::internal::eid_bad_tagged_msg_cast); - } - return h->value(); - } - template - bool variant_is_a() const { return dynamic_cast*>(punned_cast(&my_space)) != NULL; } - - bool variant_is_default_constructed() const {return variant_is_a();} - - ~variant() { - WrapperBase *h = punned_cast(&my_space); - h->~WrapperBase(); - } - }; //class variant - - TagType my_tag; - variant my_msg; - -public: - tagged_msg(): my_tag(TagType(~0)), my_msg(){} - - template - tagged_msg(T const &index, R const &value) : my_tag(index), my_msg(value) {} - - #if __TBB_CONST_REF_TO_ARRAY_TEMPLATE_PARAM_BROKEN - template - tagged_msg(T const &index, R (&value)[N]) : my_tag(index), my_msg(value) {} - #endif - - void set_tag(TagType const &index) {my_tag = index;} - TagType tag() const {return my_tag;} - - template - const V& cast_to() const {return my_msg.template variant_cast_to();} - - template - bool is_a() const {return my_msg.template variant_is_a();} - - bool is_default_constructed() const {return my_msg.variant_is_default_constructed();} -}; //class tagged_msg - -// template to simplify cast and test for tagged_msg in template contexts -template -const V& cast_to(T const &t) { return t.template cast_to(); } - -template -bool is_a(T const &t) { return t.template is_a(); } - -enum op_stat { WAIT = 0, SUCCEEDED, FAILED }; - -} // namespace internal - -#endif /* __TBB__flow_graph_types_impl_H */ diff --git a/src/3rd party/tbb/internal/_mutex_padding.h b/src/3rd party/tbb/internal/_mutex_padding.h deleted file mode 100644 index 09fccd44ba..0000000000 --- a/src/3rd party/tbb/internal/_mutex_padding.h +++ /dev/null @@ -1,98 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_mutex_padding_H -#define __TBB_mutex_padding_H - -// wrapper for padding mutexes to be alone on a cache line, without requiring they be allocated -// from a pool. Because we allow them to be defined anywhere they must be two cache lines in size. - - -namespace tbb { -namespace interface7 { -namespace internal { - -static const size_t cache_line_size = 64; - -// Pad a mutex to occupy a number of full cache lines sufficient to avoid false sharing -// with other data; space overhead is up to 2*cache_line_size-1. -template class padded_mutex; - -template -class padded_mutex : tbb::internal::mutex_copy_deprecated_and_disabled { - typedef long pad_type; - pad_type my_pad[((sizeof(Mutex)+cache_line_size-1)/cache_line_size+1)*cache_line_size/sizeof(pad_type)]; - - Mutex *impl() { return (Mutex *)((uintptr_t(this)|(cache_line_size-1))+1);} - -public: - static const bool is_rw_mutex = Mutex::is_rw_mutex; - static const bool is_recursive_mutex = Mutex::is_recursive_mutex; - static const bool is_fair_mutex = Mutex::is_fair_mutex; - - padded_mutex() { new(impl()) Mutex(); } - ~padded_mutex() { impl()->~Mutex(); } - - //! Represents acquisition of a mutex. - class scoped_lock : tbb::internal::no_copy { - typename Mutex::scoped_lock my_scoped_lock; - public: - scoped_lock() : my_scoped_lock() {} - scoped_lock( padded_mutex& m ) : my_scoped_lock(*m.impl()) { } - ~scoped_lock() { } - - void acquire( padded_mutex& m ) { my_scoped_lock.acquire(*m.impl()); } - bool try_acquire( padded_mutex& m ) { return my_scoped_lock.try_acquire(*m.impl()); } - void release() { my_scoped_lock.release(); } - }; -}; - -template -class padded_mutex : tbb::internal::mutex_copy_deprecated_and_disabled { - typedef long pad_type; - pad_type my_pad[((sizeof(Mutex)+cache_line_size-1)/cache_line_size+1)*cache_line_size/sizeof(pad_type)]; - - Mutex *impl() { return (Mutex *)((uintptr_t(this)|(cache_line_size-1))+1);} - -public: - static const bool is_rw_mutex = Mutex::is_rw_mutex; - static const bool is_recursive_mutex = Mutex::is_recursive_mutex; - static const bool is_fair_mutex = Mutex::is_fair_mutex; - - padded_mutex() { new(impl()) Mutex(); } - ~padded_mutex() { impl()->~Mutex(); } - - //! Represents acquisition of a mutex. - class scoped_lock : tbb::internal::no_copy { - typename Mutex::scoped_lock my_scoped_lock; - public: - scoped_lock() : my_scoped_lock() {} - scoped_lock( padded_mutex& m, bool write = true ) : my_scoped_lock(*m.impl(),write) { } - ~scoped_lock() { } - - void acquire( padded_mutex& m, bool write = true ) { my_scoped_lock.acquire(*m.impl(),write); } - bool try_acquire( padded_mutex& m, bool write = true ) { return my_scoped_lock.try_acquire(*m.impl(),write); } - bool upgrade_to_writer() { return my_scoped_lock.upgrade_to_writer(); } - bool downgrade_to_reader() { return my_scoped_lock.downgrade_to_reader(); } - void release() { my_scoped_lock.release(); } - }; -}; - -} // namespace internal -} // namespace interface7 -} // namespace tbb - -#endif /* __TBB_mutex_padding_H */ diff --git a/src/3rd party/tbb/internal/_node_handle_impl.h b/src/3rd party/tbb/internal/_node_handle_impl.h deleted file mode 100644 index a910b5fa5c..0000000000 --- a/src/3rd party/tbb/internal/_node_handle_impl.h +++ /dev/null @@ -1,168 +0,0 @@ -/* - Copyright (c) 2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_node_handle_H -#define __TBB_node_handle_H - -#include "_allocator_traits.h" -#include "../tbb_config.h" - - -namespace tbb { - -// This classes must be declared here for correct friendly relationship -// TODO: Consider creation some internal class to access node_handle private fields without any friendly classes -namespace interface5 { -namespace internal { - template - class split_ordered_list; - template - class concurrent_unordered_base; -} -} - -namespace interface10{ -namespace internal { - template - class concurrent_skip_list; -} -} - -namespace internal { - -template -class node_handle_base { -public: - typedef Allocator allocator_type; -protected: - typedef Node node; - typedef tbb::internal::allocator_traits traits_type; -public: - - node_handle_base() : my_node(NULL), my_allocator() {} - node_handle_base(node_handle_base&& nh) : my_node(nh.my_node), - my_allocator(std::move(nh.my_allocator)) { - nh.my_node = NULL; - } - - bool empty() const { return my_node == NULL; } - explicit operator bool() const { return my_node != NULL; } - - ~node_handle_base() { internal_destroy(); } - - node_handle_base& operator=(node_handle_base&& nh) { - internal_destroy(); - my_node = nh.my_node; - typedef typename traits_type::propagate_on_container_move_assignment pocma_type; - tbb::internal::allocator_move_assignment(my_allocator, nh.my_allocator, pocma_type()); - nh.deactivate(); - return *this; - } - - void swap(node_handle_base& nh) { - std::swap(my_node, nh.my_node); - typedef typename traits_type::propagate_on_container_swap pocs_type; - tbb::internal::allocator_swap(my_allocator, nh.my_allocator, pocs_type()); - } - - allocator_type get_allocator() const { - return my_allocator; - } - -protected: - node_handle_base(node* n) : my_node(n) {} - - void internal_destroy() { - if(my_node) { - traits_type::destroy(my_allocator, my_node->storage()); - typename tbb::internal::allocator_rebind::type node_allocator; - node_allocator.deallocate(my_node, 1); - } - } - - void deactivate() { my_node = NULL; } - - node* my_node; - allocator_type my_allocator; -}; - -// node handle for maps -template -class node_handle : public node_handle_base { - typedef node_handle_base base_type; -public: - typedef Key key_type; - typedef typename Value::second_type mapped_type; - typedef typename base_type::allocator_type allocator_type; - - node_handle() : base_type() {} - - key_type& key() const { - __TBB_ASSERT(!this->empty(), "Cannot get key from the empty node_type object"); - return *const_cast(&(this->my_node->value().first)); - } - - mapped_type& mapped() const { - __TBB_ASSERT(!this->empty(), "Cannot get mapped value from the empty node_type object"); - return this->my_node->value().second; - } - -private: - template - friend class tbb::interface5::internal::split_ordered_list; - - template - friend class tbb::interface5::internal::concurrent_unordered_base; - - template - friend class tbb::interface10::internal::concurrent_skip_list; - - node_handle(typename base_type::node* n) : base_type(n) {} -}; - -// node handle for sets -template -class node_handle : public node_handle_base { - typedef node_handle_base base_type; -public: - typedef Key value_type; - typedef typename base_type::allocator_type allocator_type; - - node_handle() : base_type() {} - - value_type& value() const { - __TBB_ASSERT(!this->empty(), "Cannot get value from the empty node_type object"); - return *const_cast(&(this->my_node->value())); - } - -private: - template - friend class tbb::interface5::internal::split_ordered_list; - - template - friend class tbb::interface5::internal::concurrent_unordered_base; - - template - friend class tbb::interface10::internal::concurrent_skip_list; - - node_handle(typename base_type::node* n) : base_type(n) {} -}; - - -}// namespace internal -}// namespace tbb - -#endif /*__TBB_node_handle_H*/ diff --git a/src/3rd party/tbb/internal/_range_iterator.h b/src/3rd party/tbb/internal/_range_iterator.h deleted file mode 100644 index 733c795f46..0000000000 --- a/src/3rd party/tbb/internal/_range_iterator.h +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_range_iterator_H -#define __TBB_range_iterator_H - -#include "../tbb_stddef.h" - -#if __TBB_CPP11_STD_BEGIN_END_PRESENT && __TBB_CPP11_AUTO_PRESENT && __TBB_CPP11_DECLTYPE_PRESENT - #include -#endif - -namespace tbb { - // iterators to first and last elements of container - namespace internal { - -#if __TBB_CPP11_STD_BEGIN_END_PRESENT && __TBB_CPP11_AUTO_PRESENT && __TBB_CPP11_DECLTYPE_PRESENT - using std::begin; - using std::end; - template - auto first(Container& c)-> decltype(begin(c)) {return begin(c);} - - template - auto first(const Container& c)-> decltype(begin(c)) {return begin(c);} - - template - auto last(Container& c)-> decltype(begin(c)) {return end(c);} - - template - auto last(const Container& c)-> decltype(begin(c)) {return end(c);} -#else - template - typename Container::iterator first(Container& c) {return c.begin();} - - template - typename Container::const_iterator first(const Container& c) {return c.begin();} - - template - typename Container::iterator last(Container& c) {return c.end();} - - template - typename Container::const_iterator last(const Container& c) {return c.end();} -#endif - - template - T* first(T (&arr) [size]) {return arr;} - - template - T* last(T (&arr) [size]) {return arr + size;} - } //namespace internal -} //namespace tbb - -#endif // __TBB_range_iterator_H diff --git a/src/3rd party/tbb/internal/_tbb_hash_compare_impl.h b/src/3rd party/tbb/internal/_tbb_hash_compare_impl.h deleted file mode 100644 index f3e5de2ee9..0000000000 --- a/src/3rd party/tbb/internal/_tbb_hash_compare_impl.h +++ /dev/null @@ -1,105 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// must be included outside namespaces. -#ifndef __TBB_tbb_hash_compare_impl_H -#define __TBB_tbb_hash_compare_impl_H - -#include - -namespace tbb { -namespace interface5 { -namespace internal { - -// Template class for hash compare -template -class hash_compare -{ -public: - typedef Hasher hasher; - typedef Key_equality key_equal; - - hash_compare() {} - - hash_compare(Hasher a_hasher) : my_hash_object(a_hasher) {} - - hash_compare(Hasher a_hasher, Key_equality a_keyeq) : my_hash_object(a_hasher), my_key_compare_object(a_keyeq) {} - - size_t operator()(const Key& key) const { - return ((size_t)my_hash_object(key)); - } - - bool operator()(const Key& key1, const Key& key2) const { - // TODO: get rid of the result invertion - return (!my_key_compare_object(key1, key2)); - } - - Hasher my_hash_object; // The hash object - Key_equality my_key_compare_object; // The equality comparator object -}; - -//! Hash multiplier -static const size_t hash_multiplier = tbb::internal::select_size_t_constant<2654435769U, 11400714819323198485ULL>::value; - -} // namespace internal - -//! Hasher functions -template -__TBB_DEPRECATED_MSG("tbb::tbb_hasher is deprecated, use std::hash") inline size_t tbb_hasher( const T& t ) { - return static_cast( t ) * internal::hash_multiplier; -} -template -__TBB_DEPRECATED_MSG("tbb::tbb_hasher is deprecated, use std::hash") inline size_t tbb_hasher( P* ptr ) { - size_t const h = reinterpret_cast( ptr ); - return (h >> 3) ^ h; -} -template -__TBB_DEPRECATED_MSG("tbb::tbb_hasher is deprecated, use std::hash") inline size_t tbb_hasher( const std::basic_string& s ) { - size_t h = 0; - for( const E* c = s.c_str(); *c; ++c ) - h = static_cast(*c) ^ (h * internal::hash_multiplier); - return h; -} -template -__TBB_DEPRECATED_MSG("tbb::tbb_hasher is deprecated, use std::hash") inline size_t tbb_hasher( const std::pair& p ) { - return tbb_hasher(p.first) ^ tbb_hasher(p.second); -} - -} // namespace interface5 -using interface5::tbb_hasher; - -// Template class for hash compare -template -class __TBB_DEPRECATED_MSG("tbb::tbb_hash is deprecated, use std::hash") tbb_hash -{ -public: - tbb_hash() {} - - size_t operator()(const Key& key) const - { - return tbb_hasher(key); - } -}; - -//! hash_compare that is default argument for concurrent_hash_map -template -struct tbb_hash_compare { - static size_t hash( const Key& a ) { return tbb_hasher(a); } - static bool equal( const Key& a, const Key& b ) { return a == b; } -}; - -} // namespace tbb -#endif /* __TBB_tbb_hash_compare_impl_H */ diff --git a/src/3rd party/tbb/internal/_tbb_trace_impl.h b/src/3rd party/tbb/internal/_tbb_trace_impl.h deleted file mode 100644 index e89ab23241..0000000000 --- a/src/3rd party/tbb/internal/_tbb_trace_impl.h +++ /dev/null @@ -1,55 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef _FGT_TBB_TRACE_IMPL_H -#define _FGT_TBB_TRACE_IMPL_H - -#include "../tbb_profiling.h" - -namespace tbb { - namespace internal { - -#if TBB_PREVIEW_ALGORITHM_TRACE - static inline void fgt_algorithm( string_index t, void *algorithm, void *parent ) { - itt_make_task_group( ITT_DOMAIN_FLOW, algorithm, ALGORITHM, parent, ALGORITHM, t ); - } - static inline void fgt_begin_algorithm( string_index t, void *algorithm ) { - itt_task_begin( ITT_DOMAIN_FLOW, algorithm, ALGORITHM, NULL, FLOW_NULL, t ); - } - static inline void fgt_end_algorithm( void * ) { - itt_task_end( ITT_DOMAIN_FLOW ); - } - static inline void fgt_alg_begin_body( string_index t, void *body, void *algorithm ) { - itt_task_begin( ITT_DOMAIN_FLOW, body, FLOW_BODY, algorithm, ALGORITHM, t ); - } - static inline void fgt_alg_end_body( void * ) { - itt_task_end( ITT_DOMAIN_FLOW ); - } - -#else // TBB_PREVIEW_ALGORITHM_TRACE - - static inline void fgt_algorithm( string_index /*t*/, void * /*algorithm*/, void * /*parent*/ ) { } - static inline void fgt_begin_algorithm( string_index /*t*/, void * /*algorithm*/ ) { } - static inline void fgt_end_algorithm( void * ) { } - static inline void fgt_alg_begin_body( string_index /*t*/, void * /*body*/, void * /*algorithm*/ ) { } - static inline void fgt_alg_end_body( void * ) { } - -#endif // TBB_PREVIEW_ALGORITHM_TRACEE - - } // namespace internal -} // namespace tbb - -#endif diff --git a/src/3rd party/tbb/internal/_tbb_windef.h b/src/3rd party/tbb/internal/_tbb_windef.h deleted file mode 100644 index 1268ba271c..0000000000 --- a/src/3rd party/tbb/internal/_tbb_windef.h +++ /dev/null @@ -1,69 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_tbb_windef_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif /* __TBB_tbb_windef_H */ - -// Check that the target Windows version has all API calls required for TBB. -// Do not increase the version in condition beyond 0x0500 without prior discussion! -#if defined(_WIN32_WINNT) && _WIN32_WINNT<0x0501 -#error TBB is unable to run on old Windows versions; _WIN32_WINNT must be 0x0501 or greater. -#endif - -#if !defined(_MT) -#error TBB requires linkage with multithreaded C/C++ runtime library. \ - Choose multithreaded DLL runtime in project settings, or use /MD[d] compiler switch. -#endif - -// Workaround for the problem with MVSC headers failing to define namespace std -namespace std { - using ::size_t; using ::ptrdiff_t; -} - -#define __TBB_STRING_AUX(x) #x -#define __TBB_STRING(x) __TBB_STRING_AUX(x) - -// Default setting of TBB_USE_DEBUG -#ifdef TBB_USE_DEBUG -# if TBB_USE_DEBUG -# if !defined(_DEBUG) -# pragma message(__FILE__ "(" __TBB_STRING(__LINE__) ") : Warning: Recommend using /MDd if compiling with TBB_USE_DEBUG!=0") -# endif -# else -# if defined(_DEBUG) -# pragma message(__FILE__ "(" __TBB_STRING(__LINE__) ") : Warning: Recommend using /MD if compiling with TBB_USE_DEBUG==0") -# endif -# endif -#endif - -#if (__TBB_BUILD || __TBBMALLOC_BUILD) && !defined(__TBB_NO_IMPLICIT_LINKAGE) -#define __TBB_NO_IMPLICIT_LINKAGE 1 -#endif - -#if _MSC_VER - #if !__TBB_NO_IMPLICIT_LINKAGE - #ifdef __TBB_LIB_NAME - #pragma comment(lib, __TBB_STRING(__TBB_LIB_NAME)) - #else - #ifdef _DEBUG - #pragma comment(lib, "tbb_debug.lib") - #else - #pragma comment(lib, "tbb.lib") - #endif - #endif - #endif -#endif diff --git a/src/3rd party/tbb/internal/_template_helpers.h b/src/3rd party/tbb/internal/_template_helpers.h deleted file mode 100644 index d31a3da31e..0000000000 --- a/src/3rd party/tbb/internal/_template_helpers.h +++ /dev/null @@ -1,284 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_template_helpers_H -#define __TBB_template_helpers_H - -#include -#include -#include "../tbb_config.h" -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT -#include -#endif -#if __TBB_CPP11_PRESENT -#include -#include // allocator_traits -#endif - -namespace tbb { namespace internal { - -//! Enables one or the other code branches -template struct enable_if {}; -template struct enable_if { typedef T type; }; - -//! Strips its template type argument from cv- and ref-qualifiers -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -//! Specialization for function pointers -template struct strip { typedef T(*type)(); }; -#if __TBB_CPP11_RVALUE_REF_PRESENT -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -template struct strip { typedef T type; }; -#endif -//! Specialization for arrays converts to a corresponding pointer -template struct strip { typedef T* type; }; -template struct strip { typedef const T* type; }; -template struct strip { typedef volatile T* type; }; -template struct strip { typedef const volatile T* type; }; - -//! Detects whether two given types are the same -template struct is_same_type { static const bool value = false; }; -template struct is_same_type { static const bool value = true; }; - -template struct is_ref { static const bool value = false; }; -template struct is_ref { static const bool value = true; }; - -//! Partial support for std::is_integral -template struct is_integral_impl { static const bool value = false; }; -template<> struct is_integral_impl { static const bool value = true; }; -template<> struct is_integral_impl { static const bool value = true; }; -#if __TBB_CPP11_PRESENT -template<> struct is_integral_impl { static const bool value = true; }; -template<> struct is_integral_impl { static const bool value = true; }; -#endif -template<> struct is_integral_impl { static const bool value = true; }; -template<> struct is_integral_impl { static const bool value = true; }; -template<> struct is_integral_impl { static const bool value = true; }; -template<> struct is_integral_impl { static const bool value = true; }; -template<> struct is_integral_impl { static const bool value = true; }; - -template -struct is_integral : is_integral_impl::type> {}; - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -//! std::void_t internal implementation (to avoid GCC < 4.7 "template aliases" absence) -template struct void_t { typedef void type; }; -#endif - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT - -// Generic SFINAE helper for expression checks, based on the idea demonstrated in ISO C++ paper n4502 -template class... Checks> -struct supports_impl { typedef std::false_type type; }; -template class... Checks> -struct supports_impl...>::type, Checks...> { typedef std::true_type type; }; - -template class... Checks> -using supports = typename supports_impl::type; - -#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT */ - -#if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - -//! Allows to store a function parameter pack as a variable and later pass it to another function -template< typename... Types > -struct stored_pack; - -template<> -struct stored_pack<> -{ - typedef stored_pack<> pack_type; - stored_pack() {} - - // Friend front-end functions - template< typename F, typename Pack > friend void call( F&& f, Pack&& p ); - template< typename Ret, typename F, typename Pack > friend Ret call_and_return( F&& f, Pack&& p ); - -protected: - // Ideally, ref-qualified non-static methods would be used, - // but that would greatly reduce the set of compilers where it works. - template< typename Ret, typename F, typename... Preceding > - static Ret call( F&& f, const pack_type& /*pack*/, Preceding&&... params ) { - return std::forward(f)( std::forward(params)... ); - } - template< typename Ret, typename F, typename... Preceding > - static Ret call( F&& f, pack_type&& /*pack*/, Preceding&&... params ) { - return std::forward(f)( std::forward(params)... ); - } -}; - -template< typename T, typename... Types > -struct stored_pack : stored_pack -{ - typedef stored_pack pack_type; - typedef stored_pack pack_remainder; - // Since lifetime of original values is out of control, copies should be made. - // Thus references should be stripped away from the deduced type. - typename strip::type leftmost_value; - - // Here rvalue references act in the same way as forwarding references, - // as long as class template parameters were deduced via forwarding references. - stored_pack( T&& t, Types&&... types ) - : pack_remainder(std::forward(types)...), leftmost_value(std::forward(t)) {} - - // Friend front-end functions - template< typename F, typename Pack > friend void call( F&& f, Pack&& p ); - template< typename Ret, typename F, typename Pack > friend Ret call_and_return( F&& f, Pack&& p ); - -protected: - template< typename Ret, typename F, typename... Preceding > - static Ret call( F&& f, pack_type& pack, Preceding&&... params ) { - return pack_remainder::template call( - std::forward(f), static_cast(pack), - std::forward(params)... , pack.leftmost_value - ); - } - template< typename Ret, typename F, typename... Preceding > - static Ret call( F&& f, const pack_type& pack, Preceding&&... params ) { - return pack_remainder::template call( - std::forward(f), static_cast(pack), - std::forward(params)... , pack.leftmost_value - ); - } - template< typename Ret, typename F, typename... Preceding > - static Ret call( F&& f, pack_type&& pack, Preceding&&... params ) { - return pack_remainder::template call( - std::forward(f), static_cast(pack), - std::forward(params)... , std::move(pack.leftmost_value) - ); - } -}; - -//! Calls the given function with arguments taken from a stored_pack -template< typename F, typename Pack > -void call( F&& f, Pack&& p ) { - strip::type::template call( std::forward(f), std::forward(p) ); -} - -template< typename Ret, typename F, typename Pack > -Ret call_and_return( F&& f, Pack&& p ) { - return strip::type::template call( std::forward(f), std::forward(p) ); -} - -template< typename... Types > -stored_pack save_pack( Types&&... types ) { - return stored_pack( std::forward(types)... ); -} - -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ - -#if __TBB_CPP14_INTEGER_SEQUENCE_PRESENT - -using std::index_sequence; -using std::make_index_sequence; - -#elif __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT - -template class index_sequence {}; - -template -struct make_index_sequence_impl : make_index_sequence_impl < N - 1, N - 1, S... > {}; - -template -struct make_index_sequence_impl <0, S...> { - using type = index_sequence; -}; - -template -using make_index_sequence = typename tbb::internal::make_index_sequence_impl::type; - -#endif /* __TBB_CPP14_INTEGER_SEQUENCE_PRESENT */ - -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -template -struct conjunction; - -template -struct conjunction - : std::conditional, First>::type {}; - -template -struct conjunction : T {}; - -template<> -struct conjunction<> : std::true_type {}; - -#endif - -#if __TBB_CPP11_PRESENT - -template< typename Iter > -using iterator_value_t = typename std::iterator_traits::value_type; - -template< typename Iter > -using iterator_key_t = typename std::remove_const::first_type>::type; - -template< typename Iter > -using iterator_mapped_t = typename iterator_value_t::second_type; - -template< typename A > using value_type = typename A::value_type; -template< typename A > using alloc_ptr_t = typename std::allocator_traits::pointer; -template< typename A > using has_allocate = decltype(std::declval&>() = std::declval().allocate(0)); -template< typename A > using has_deallocate = decltype(std::declval().deallocate(std::declval>(), 0)); - -// value_type should be checked first because it can be used in other checks (via allocator_traits) -template< typename T > -using is_allocator = supports; - -#if __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT - -template< typename T > -static constexpr bool is_allocator_v = is_allocator::value; - -#endif /*__TBB_CPP14_VARIABLE_TEMPLATES */ - -template< std::size_t N, typename... Args > -struct pack_element { - using type = void; -}; - -template< std::size_t N, typename T, typename... Args > -struct pack_element { - using type = typename pack_element::type; -}; - -template< typename T, typename... Args > -struct pack_element<0, T, Args...> { - using type = T; -}; - -template< std::size_t N, typename... Args > -using pack_element_t = typename pack_element::type; - -// Helper alias for heterogeneous lookup functions in containers -// template parameter K and std::conditional are needed to provide immediate context -// and postpone getting is_trasparent from the compare functor until method instantiation. -template -using is_transparent = typename std::conditional::type::is_transparent; - -#endif /* __TBB_CPP11_PRESENT */ - -} } // namespace internal, namespace tbb - -#endif /* __TBB_template_helpers_H */ diff --git a/src/3rd party/tbb/internal/_x86_eliding_mutex_impl.h b/src/3rd party/tbb/internal/_x86_eliding_mutex_impl.h deleted file mode 100644 index a03e463fd4..0000000000 --- a/src/3rd party/tbb/internal/_x86_eliding_mutex_impl.h +++ /dev/null @@ -1,144 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB__x86_eliding_mutex_impl_H -#define __TBB__x86_eliding_mutex_impl_H - -#ifndef __TBB_spin_mutex_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#if ( __TBB_x86_32 || __TBB_x86_64 ) - -namespace tbb { -namespace interface7 { -namespace internal { - -template -class padded_mutex; - -//! An eliding lock that occupies a single byte. -/** A x86_eliding_mutex is an HLE-enabled spin mutex. It is recommended to - put the mutex on a cache line that is not shared by the data it protects. - It should be used for locking short critical sections where the lock is - contended but the data it protects are not. If zero-initialized, the - mutex is considered unheld. - @ingroup synchronization */ -class x86_eliding_mutex : tbb::internal::mutex_copy_deprecated_and_disabled { - //! 0 if lock is released, 1 if lock is acquired. - __TBB_atomic_flag flag; - - friend class padded_mutex; - -public: - //! Construct unacquired lock. - /** Equivalent to zero-initialization of *this. */ - x86_eliding_mutex() : flag(0) {} - -// bug in gcc 3.x.x causes syntax error in spite of the friend declaration above. -// Make the scoped_lock public in that case. -#if __TBB_USE_X86_ELIDING_MUTEX || __TBB_GCC_VERSION < 40000 -#else - // by default we will not provide the scoped_lock interface. The user - // should use the padded version of the mutex. scoped_lock is used in - // padded_mutex template. -private: -#endif - // scoped_lock in padded_mutex<> is the interface to use. - //! Represents acquisition of a mutex. - class scoped_lock : tbb::internal::no_copy { - private: - //! Points to currently held mutex, or NULL if no lock is held. - x86_eliding_mutex* my_mutex; - - public: - //! Construct without acquiring a mutex. - scoped_lock() : my_mutex(NULL) {} - - //! Construct and acquire lock on a mutex. - scoped_lock( x86_eliding_mutex& m ) : my_mutex(NULL) { acquire(m); } - - //! Acquire lock. - void acquire( x86_eliding_mutex& m ) { - __TBB_ASSERT( !my_mutex, "already holding a lock" ); - - my_mutex=&m; - my_mutex->lock(); - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_acquire( x86_eliding_mutex& m ) { - __TBB_ASSERT( !my_mutex, "already holding a lock" ); - - bool result = m.try_lock(); - if( result ) { - my_mutex = &m; - } - return result; - } - - //! Release lock - void release() { - __TBB_ASSERT( my_mutex, "release on scoped_lock that is not holding a lock" ); - - my_mutex->unlock(); - my_mutex = NULL; - } - - //! Destroy lock. If holding a lock, releases the lock first. - ~scoped_lock() { - if( my_mutex ) { - release(); - } - } - }; -#if __TBB_USE_X86_ELIDING_MUTEX || __TBB_GCC_VERSION < 40000 -#else -public: -#endif /* __TBB_USE_X86_ELIDING_MUTEX */ - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = false; - - // ISO C++0x compatibility methods - - //! Acquire lock - void lock() { - __TBB_LockByteElided(flag); - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_lock() { - return __TBB_TryLockByteElided(flag); - } - - //! Release lock - void unlock() { - __TBB_UnlockByteElided( flag ); - } -}; // end of x86_eliding_mutex - -} // namespace internal -} // namespace interface7 -} // namespace tbb - -#endif /* ( __TBB_x86_32 || __TBB_x86_64 ) */ - -#endif /* __TBB__x86_eliding_mutex_impl_H */ diff --git a/src/3rd party/tbb/internal/_x86_rtm_rw_mutex_impl.h b/src/3rd party/tbb/internal/_x86_rtm_rw_mutex_impl.h deleted file mode 100644 index 0003abac9f..0000000000 --- a/src/3rd party/tbb/internal/_x86_rtm_rw_mutex_impl.h +++ /dev/null @@ -1,223 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB__x86_rtm_rw_mutex_impl_H -#define __TBB__x86_rtm_rw_mutex_impl_H - -#ifndef __TBB_spin_rw_mutex_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#if __TBB_TSX_AVAILABLE - -#include "../tbb_stddef.h" -#include "../tbb_machine.h" -#include "../tbb_profiling.h" -#include "../spin_rw_mutex.h" - -namespace tbb { -namespace interface8 { -namespace internal { - -enum RTM_type { - RTM_not_in_mutex, - RTM_transacting_reader, - RTM_transacting_writer, - RTM_real_reader, - RTM_real_writer -}; - -static const unsigned long speculation_granularity = 64; - -//! Fast, unfair, spinning speculation-enabled reader-writer lock with backoff and -// writer-preference -/** @ingroup synchronization */ -class x86_rtm_rw_mutex: private spin_rw_mutex { -#if __TBB_USE_X86_RTM_RW_MUTEX || __TBB_GCC_VERSION < 40000 -// bug in gcc 3.x.x causes syntax error in spite of the friend declaration below. -// Make the scoped_lock public in that case. -public: -#else -private: -#endif - friend class interface7::internal::padded_mutex; - class scoped_lock; // should be private - friend class scoped_lock; -private: - //! @cond INTERNAL - - //! Internal construct unacquired mutex. - void __TBB_EXPORTED_METHOD internal_construct(); - - //! Internal acquire write lock. - // only_speculate == true if we're doing a try_lock, else false. - void __TBB_EXPORTED_METHOD internal_acquire_writer(x86_rtm_rw_mutex::scoped_lock&, bool only_speculate=false); - - //! Internal acquire read lock. - // only_speculate == true if we're doing a try_lock, else false. - void __TBB_EXPORTED_METHOD internal_acquire_reader(x86_rtm_rw_mutex::scoped_lock&, bool only_speculate=false); - - //! Internal upgrade reader to become a writer. - bool __TBB_EXPORTED_METHOD internal_upgrade( x86_rtm_rw_mutex::scoped_lock& ); - - //! Out of line code for downgrading a writer to a reader. - bool __TBB_EXPORTED_METHOD internal_downgrade( x86_rtm_rw_mutex::scoped_lock& ); - - //! Internal try_acquire write lock. - bool __TBB_EXPORTED_METHOD internal_try_acquire_writer( x86_rtm_rw_mutex::scoped_lock& ); - - //! Internal release lock. - void __TBB_EXPORTED_METHOD internal_release( x86_rtm_rw_mutex::scoped_lock& ); - - static x86_rtm_rw_mutex* internal_get_mutex( const spin_rw_mutex::scoped_lock& lock ) - { - return static_cast( lock.mutex ); - } - static void internal_set_mutex( spin_rw_mutex::scoped_lock& lock, spin_rw_mutex* mtx ) - { - lock.mutex = mtx; - } - //! @endcond -public: - //! Construct unacquired mutex. - x86_rtm_rw_mutex() { - w_flag = false; -#if TBB_USE_THREADING_TOOLS - internal_construct(); -#endif - } - -#if TBB_USE_ASSERT - //! Empty destructor. - ~x86_rtm_rw_mutex() {} -#endif /* TBB_USE_ASSERT */ - - // Mutex traits - static const bool is_rw_mutex = true; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = false; - -#if __TBB_USE_X86_RTM_RW_MUTEX || __TBB_GCC_VERSION < 40000 -#else - // by default we will not provide the scoped_lock interface. The user - // should use the padded version of the mutex. scoped_lock is used in - // padded_mutex template. -private: -#endif - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - // Speculation-enabled scoped lock for spin_rw_mutex - // The idea is to be able to reuse the acquire/release methods of spin_rw_mutex - // and its scoped lock wherever possible. The only way to use a speculative lock is to use - // a scoped_lock. (because transaction_state must be local) - - class scoped_lock : tbb::internal::no_copy { - friend class x86_rtm_rw_mutex; - spin_rw_mutex::scoped_lock my_scoped_lock; - - RTM_type transaction_state; - - public: - //! Construct lock that has not acquired a mutex. - /** Equivalent to zero-initialization of *this. */ - scoped_lock() : my_scoped_lock(), transaction_state(RTM_not_in_mutex) { - } - - //! Acquire lock on given mutex. - scoped_lock( x86_rtm_rw_mutex& m, bool write = true ) : my_scoped_lock(), - transaction_state(RTM_not_in_mutex) { - acquire(m, write); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if(transaction_state != RTM_not_in_mutex) release(); - } - - //! Acquire lock on given mutex. - void acquire( x86_rtm_rw_mutex& m, bool write = true ) { - if( write ) m.internal_acquire_writer(*this); - else m.internal_acquire_reader(*this); - } - - //! Release lock - void release() { - x86_rtm_rw_mutex* mutex = x86_rtm_rw_mutex::internal_get_mutex(my_scoped_lock); - __TBB_ASSERT( mutex, "lock is not acquired" ); - __TBB_ASSERT( transaction_state!=RTM_not_in_mutex, "lock is not acquired" ); - return mutex->internal_release(*this); - } - - //! Upgrade reader to become a writer. - /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ - bool upgrade_to_writer() { - x86_rtm_rw_mutex* mutex = x86_rtm_rw_mutex::internal_get_mutex(my_scoped_lock); - __TBB_ASSERT( mutex, "lock is not acquired" ); - if (transaction_state == RTM_transacting_writer || transaction_state == RTM_real_writer) - return true; // Already a writer - return mutex->internal_upgrade(*this); - } - - //! Downgrade writer to become a reader. - /** Returns whether the downgrade happened without releasing and re-acquiring the lock */ - bool downgrade_to_reader() { - x86_rtm_rw_mutex* mutex = x86_rtm_rw_mutex::internal_get_mutex(my_scoped_lock); - __TBB_ASSERT( mutex, "lock is not acquired" ); - if (transaction_state == RTM_transacting_reader || transaction_state == RTM_real_reader) - return true; // Already a reader - return mutex->internal_downgrade(*this); - } - - //! Attempt to acquire mutex. - /** returns true if successful. */ - bool try_acquire( x86_rtm_rw_mutex& m, bool write = true ) { -#if TBB_USE_ASSERT - x86_rtm_rw_mutex* mutex = x86_rtm_rw_mutex::internal_get_mutex(my_scoped_lock); - __TBB_ASSERT( !mutex, "lock is already acquired" ); -#endif - // have to assign m to our mutex. - // cannot set the mutex, because try_acquire in spin_rw_mutex depends on it being NULL. - if(write) return m.internal_try_acquire_writer(*this); - // speculatively acquire the lock. If this fails, do try_acquire on the spin_rw_mutex. - m.internal_acquire_reader(*this, /*only_speculate=*/true); - if(transaction_state == RTM_transacting_reader) return true; - if( my_scoped_lock.try_acquire(m, false)) { - transaction_state = RTM_real_reader; - return true; - } - return false; - } - - }; // class x86_rtm_rw_mutex::scoped_lock - - // ISO C++0x compatibility methods not provided because we cannot maintain - // state about whether a thread is in a transaction. - -private: - char pad[speculation_granularity-sizeof(spin_rw_mutex)]; // padding - - // If true, writer holds the spin_rw_mutex. - tbb::atomic w_flag; // want this on a separate cache line - -}; // x86_rtm_rw_mutex - -} // namespace internal -} // namespace interface8 -} // namespace tbb - -#endif /* __TBB_TSX_AVAILABLE */ -#endif /* __TBB__x86_rtm_rw_mutex_impl_H */ diff --git a/src/3rd party/tbb/iterators.h b/src/3rd party/tbb/iterators.h deleted file mode 100644 index 39f21ac0a4..0000000000 --- a/src/3rd party/tbb/iterators.h +++ /dev/null @@ -1,326 +0,0 @@ -/* - Copyright (c) 2017-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_iterators_H -#define __TBB_iterators_H - -#include -#include - -#include "tbb_config.h" -#include "tbb_stddef.h" - -#if __TBB_CPP11_PRESENT - -#include - -namespace tbb { - -template -class counting_iterator { - __TBB_STATIC_ASSERT(std::numeric_limits::is_integer, "Cannot instantiate counting_iterator with a non-integer type"); -public: - typedef typename std::make_signed::type difference_type; - typedef IntType value_type; - typedef const IntType* pointer; - typedef const IntType& reference; - typedef std::random_access_iterator_tag iterator_category; - - counting_iterator() : my_counter() {} - explicit counting_iterator(IntType init) : my_counter(init) {} - - reference operator*() const { return my_counter; } - value_type operator[](difference_type i) const { return *(*this + i); } - - difference_type operator-(const counting_iterator& it) const { return my_counter - it.my_counter; } - - counting_iterator& operator+=(difference_type forward) { my_counter += forward; return *this; } - counting_iterator& operator-=(difference_type backward) { return *this += -backward; } - counting_iterator& operator++() { return *this += 1; } - counting_iterator& operator--() { return *this -= 1; } - - counting_iterator operator++(int) { - counting_iterator it(*this); - ++(*this); - return it; - } - counting_iterator operator--(int) { - counting_iterator it(*this); - --(*this); - return it; - } - - counting_iterator operator-(difference_type backward) const { return counting_iterator(my_counter - backward); } - counting_iterator operator+(difference_type forward) const { return counting_iterator(my_counter + forward); } - friend counting_iterator operator+(difference_type forward, const counting_iterator it) { return it + forward; } - - bool operator==(const counting_iterator& it) const { return *this - it == 0; } - bool operator!=(const counting_iterator& it) const { return !(*this == it); } - bool operator<(const counting_iterator& it) const {return *this - it < 0; } - bool operator>(const counting_iterator& it) const { return it < *this; } - bool operator<=(const counting_iterator& it) const { return !(*this > it); } - bool operator>=(const counting_iterator& it) const { return !(*this < it); } - -private: - IntType my_counter; -}; -} //namespace tbb - - -#include - -#include "internal/_template_helpers.h" // index_sequence, make_index_sequence - -namespace tbb { -namespace internal { - -template -struct tuple_util { - template - static void increment(TupleType& it, DifferenceType forward) { - std::get(it) += forward; - tuple_util::increment(it, forward); - } - template - static bool check_sync(const TupleType& it1, const TupleType& it2, DifferenceType val) { - if(std::get(it1) - std::get(it2) != val) - return false; - return tuple_util::check_sync(it1, it2, val); - } -}; - -template<> -struct tuple_util<0> { - template - static void increment(TupleType&, DifferenceType) {} - template - static bool check_sync(const TupleType&, const TupleType&, DifferenceType) { return true;} -}; - -template -struct make_references { - template - TupleReturnType operator()(const TupleType& t, tbb::internal::index_sequence) { - return std::tie( *std::get(t)... ); - } -}; - -// A simple wrapper over a tuple of references. -// The class is designed to hold a temporary tuple of reference -// after dereferencing a zip_iterator; in particular, it is needed -// to swap these rvalue tuples. Any other usage is not supported. -template -struct tuplewrapper : public std::tuple::value, T&&>::type...> { - // In the context of this class, T is a reference, so T&& is a "forwarding reference" - typedef std::tuple base_type; - // Construct from the result of std::tie - tuplewrapper(const base_type& in) : base_type(in) {} -#if __INTEL_COMPILER - // ICC cannot generate copy ctor & assignment - tuplewrapper(const tuplewrapper& rhs) : base_type(rhs) {} - tuplewrapper& operator=(const tuplewrapper& rhs) { - *this = base_type(rhs); - return *this; - } -#endif - // Assign any tuple convertible to std::tuple: *it = a_tuple; - template - tuplewrapper& operator=(const std::tuple& other) { - base_type::operator=(other); - return *this; - } -#if _LIBCPP_VERSION - // (Necessary for libc++ tuples) Convert to a tuple of values: v = *it; - operator std::tuple::type...>() { return base_type(*this); } -#endif - // Swap rvalue tuples: swap(*it1,*it2); - friend void swap(tuplewrapper&& a, tuplewrapper&& b) { - std::swap(a,b); - } -}; - -} //namespace internal - -template -class zip_iterator { - __TBB_STATIC_ASSERT(sizeof...(Types)>0, "Cannot instantiate zip_iterator with empty template parameter pack"); - static const std::size_t num_types = sizeof...(Types); - typedef std::tuple it_types; -public: - typedef typename std::make_signed::type difference_type; - typedef std::tuple::value_type...> value_type; -#if __INTEL_COMPILER && __INTEL_COMPILER < 1800 && _MSC_VER - typedef std::tuple::reference...> reference; -#else - typedef tbb::internal::tuplewrapper::reference...> reference; -#endif - typedef std::tuple::pointer...> pointer; - typedef std::random_access_iterator_tag iterator_category; - - zip_iterator() : my_it() {} - explicit zip_iterator(Types... args) : my_it(std::make_tuple(args...)) {} - zip_iterator(const zip_iterator& input) : my_it(input.my_it) {} - zip_iterator& operator=(const zip_iterator& input) { - my_it = input.my_it; - return *this; - } - - reference operator*() const { - return tbb::internal::make_references()(my_it, tbb::internal::make_index_sequence()); - } - reference operator[](difference_type i) const { return *(*this + i); } - - difference_type operator-(const zip_iterator& it) const { - __TBB_ASSERT(internal::tuple_util::check_sync(my_it, it.my_it, std::get<0>(my_it) - std::get<0>(it.my_it)), - "Components of zip_iterator are not synchronous"); - return std::get<0>(my_it) - std::get<0>(it.my_it); - } - - zip_iterator& operator+=(difference_type forward) { - internal::tuple_util::increment(my_it, forward); - return *this; - } - zip_iterator& operator-=(difference_type backward) { return *this += -backward; } - zip_iterator& operator++() { return *this += 1; } - zip_iterator& operator--() { return *this -= 1; } - - zip_iterator operator++(int) { - zip_iterator it(*this); - ++(*this); - return it; - } - zip_iterator operator--(int) { - zip_iterator it(*this); - --(*this); - return it; - } - - zip_iterator operator-(difference_type backward) const { - zip_iterator it(*this); - return it -= backward; - } - zip_iterator operator+(difference_type forward) const { - zip_iterator it(*this); - return it += forward; - } - friend zip_iterator operator+(difference_type forward, const zip_iterator& it) { return it + forward; } - - bool operator==(const zip_iterator& it) const { - return *this - it == 0; - } - it_types base() const { return my_it; } - - bool operator!=(const zip_iterator& it) const { return !(*this == it); } - bool operator<(const zip_iterator& it) const { return *this - it < 0; } - bool operator>(const zip_iterator& it) const { return it < *this; } - bool operator<=(const zip_iterator& it) const { return !(*this > it); } - bool operator>=(const zip_iterator& it) const { return !(*this < it); } -private: - it_types my_it; -}; - -template -zip_iterator make_zip_iterator(T... args) { return zip_iterator(args...); } - -template -class transform_iterator { -public: - typedef typename std::iterator_traits::value_type value_type; - typedef typename std::iterator_traits::difference_type difference_type; -#if __TBB_CPP17_INVOKE_RESULT_PRESENT - typedef typename std::invoke_result::reference>::type reference; -#else - typedef typename std::result_of::reference)>::type reference; -#endif - typedef typename std::iterator_traits::pointer pointer; - typedef typename std::random_access_iterator_tag iterator_category; - - transform_iterator(Iter it, UnaryFunc unary_func) : my_it(it), my_unary_func(unary_func) { - __TBB_STATIC_ASSERT((std::is_same::iterator_category, - std::random_access_iterator_tag>::value), "Random access iterator required."); - } - transform_iterator(const transform_iterator& input) : my_it(input.my_it), my_unary_func(input.my_unary_func) { } - transform_iterator& operator=(const transform_iterator& input) { - my_it = input.my_it; - return *this; - } - reference operator*() const { - return my_unary_func(*my_it); - } - reference operator[](difference_type i) const { - return *(*this + i); - } - transform_iterator& operator++() { - ++my_it; - return *this; - } - transform_iterator& operator--() { - --my_it; - return *this; - } - transform_iterator operator++(int) { - transform_iterator it(*this); - ++(*this); - return it; - } - transform_iterator operator--(int) { - transform_iterator it(*this); - --(*this); - return it; - } - transform_iterator operator+(difference_type forward) const { - return { my_it + forward, my_unary_func }; - } - transform_iterator operator-(difference_type backward) const { - return { my_it - backward, my_unary_func }; - } - transform_iterator& operator+=(difference_type forward) { - my_it += forward; - return *this; - } - transform_iterator& operator-=(difference_type backward) { - my_it -= backward; - return *this; - } - friend transform_iterator operator+(difference_type forward, const transform_iterator& it) { - return it + forward; - } - difference_type operator-(const transform_iterator& it) const { - return my_it - it.my_it; - } - bool operator==(const transform_iterator& it) const { return *this - it == 0; } - bool operator!=(const transform_iterator& it) const { return !(*this == it); } - bool operator<(const transform_iterator& it) const { return *this - it < 0; } - bool operator>(const transform_iterator& it) const { return it < *this; } - bool operator<=(const transform_iterator& it) const { return !(*this > it); } - bool operator>=(const transform_iterator& it) const { return !(*this < it); } - - Iter base() const { return my_it; } -private: - Iter my_it; - const UnaryFunc my_unary_func; -}; - -template -transform_iterator make_transform_iterator(Iter it, UnaryFunc unary_func) { - return transform_iterator(it, unary_func); -} - -} //namespace tbb - -#endif //__TBB_CPP11_PRESENT - -#endif /* __TBB_iterators_H */ diff --git a/src/3rd party/tbb/machine/gcc_arm.h b/src/3rd party/tbb/machine/gcc_arm.h deleted file mode 100644 index 40118e8740..0000000000 --- a/src/3rd party/tbb/machine/gcc_arm.h +++ /dev/null @@ -1,216 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -/* - Platform isolation layer for the ARMv7-a architecture. -*/ - -#ifndef __TBB_machine_H -#error Do not include this file directly; include tbb_machine.h instead -#endif - -#if __ARM_ARCH_7A__ - -#include -#include - -#define __TBB_WORDSIZE 4 - -// Traditionally ARM is little-endian. -// Note that, since only the layout of aligned 32-bit words is of interest, -// any apparent PDP-endianness of 32-bit words at half-word alignment or -// any little-endian ordering of big-endian 32-bit words in 64-bit quantities -// may be disregarded for this setting. -#if __BIG_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_BIG -#elif __LITTLE_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE -#elif defined(__BYTE_ORDER__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_UNSUPPORTED -#else - #define __TBB_ENDIANNESS __TBB_ENDIAN_DETECT -#endif - - -#define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") -#define __TBB_full_memory_fence() __asm__ __volatile__("dmb ish": : :"memory") -#define __TBB_control_consistency_helper() __TBB_full_memory_fence() -#define __TBB_acquire_consistency_helper() __TBB_full_memory_fence() -#define __TBB_release_consistency_helper() __TBB_full_memory_fence() - -//-------------------------------------------------- -// Compare and swap -//-------------------------------------------------- - -/** - * Atomic CAS for 32 bit values, if *ptr==comparand, then *ptr=value, returns *ptr - * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand - * @param value value to assign *ptr to if *ptr==comparand - * @param comparand value to compare with *ptr - * @return value originally in memory at ptr, regardless of success -*/ -static inline int32_t __TBB_machine_cmpswp4(volatile void *ptr, int32_t value, int32_t comparand ) -{ - int32_t oldval, res; - - __TBB_full_memory_fence(); - - do { - __asm__ __volatile__( - "ldrex %1, [%3]\n" - "mov %0, #0\n" - "cmp %1, %4\n" - "it eq\n" - "strexeq %0, %5, [%3]\n" - : "=&r" (res), "=&r" (oldval), "+Qo" (*(volatile int32_t*)ptr) - : "r" ((volatile int32_t *)ptr), "Ir" (comparand), "r" (value) - : "cc"); - } while (res); - - __TBB_full_memory_fence(); - - return oldval; -} - -/** - * Atomic CAS for 64 bit values, if *ptr==comparand, then *ptr=value, returns *ptr - * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand - * @param value value to assign *ptr to if *ptr==comparand - * @param comparand value to compare with *ptr - * @return value originally in memory at ptr, regardless of success - */ -static inline int64_t __TBB_machine_cmpswp8(volatile void *ptr, int64_t value, int64_t comparand ) -{ - int64_t oldval; - int32_t res; - - __TBB_full_memory_fence(); - - do { - __asm__ __volatile__( - "mov %0, #0\n" - "ldrexd %1, %H1, [%3]\n" - "cmp %1, %4\n" - "it eq\n" - "cmpeq %H1, %H4\n" - "it eq\n" - "strexdeq %0, %5, %H5, [%3]" - : "=&r" (res), "=&r" (oldval), "+Qo" (*(volatile int64_t*)ptr) - : "r" ((volatile int64_t *)ptr), "r" (comparand), "r" (value) - : "cc"); - } while (res); - - __TBB_full_memory_fence(); - - return oldval; -} - -static inline int32_t __TBB_machine_fetchadd4(volatile void* ptr, int32_t addend) -{ - unsigned long tmp; - int32_t result, tmp2; - - __TBB_full_memory_fence(); - - __asm__ __volatile__( -"1: ldrex %0, [%4]\n" -" add %3, %0, %5\n" -" strex %1, %3, [%4]\n" -" cmp %1, #0\n" -" bne 1b\n" - : "=&r" (result), "=&r" (tmp), "+Qo" (*(volatile int32_t*)ptr), "=&r"(tmp2) - : "r" ((volatile int32_t *)ptr), "Ir" (addend) - : "cc"); - - __TBB_full_memory_fence(); - - return result; -} - -static inline int64_t __TBB_machine_fetchadd8(volatile void *ptr, int64_t addend) -{ - unsigned long tmp; - int64_t result, tmp2; - - __TBB_full_memory_fence(); - - __asm__ __volatile__( -"1: ldrexd %0, %H0, [%4]\n" -" adds %3, %0, %5\n" -" adc %H3, %H0, %H5\n" -" strexd %1, %3, %H3, [%4]\n" -" cmp %1, #0\n" -" bne 1b" - : "=&r" (result), "=&r" (tmp), "+Qo" (*(volatile int64_t*)ptr), "=&r"(tmp2) - : "r" ((volatile int64_t *)ptr), "r" (addend) - : "cc"); - - - __TBB_full_memory_fence(); - - return result; -} - -namespace tbb { -namespace internal { - template - struct machine_load_store_relaxed { - static inline T load ( const volatile T& location ) { - const T value = location; - - /* - * An extra memory barrier is required for errata #761319 - * Please see http://infocenter.arm.com/help/topic/com.arm.doc.uan0004a - */ - __TBB_acquire_consistency_helper(); - return value; - } - - static inline void store ( volatile T& location, T value ) { - location = value; - } - }; -}} // namespaces internal, tbb - -// Machine specific atomic operations - -#define __TBB_CompareAndSwap4(P,V,C) __TBB_machine_cmpswp4(P,V,C) -#define __TBB_CompareAndSwap8(P,V,C) __TBB_machine_cmpswp8(P,V,C) - -// Use generics for some things -#define __TBB_USE_GENERIC_PART_WORD_CAS 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_ADD 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_STORE 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_DWORD_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 -#elif defined __aarch64__ -// Generic gcc implementations are fine for ARMv8-a except __TBB_PAUSE. -#include "gcc_generic.h" -#else -#error compilation requires an ARMv7-a or ARMv8-a architecture. -#endif // __ARM_ARCH_7A__ - -inline void __TBB_machine_pause (int32_t delay) -{ - while(delay>0) - { - __asm__ __volatile__("yield" ::: "memory"); - delay--; - } -} -#define __TBB_Pause(V) __TBB_machine_pause(V) diff --git a/src/3rd party/tbb/machine/gcc_generic.h b/src/3rd party/tbb/machine/gcc_generic.h deleted file mode 100644 index cbf8d99324..0000000000 --- a/src/3rd party/tbb/machine/gcc_generic.h +++ /dev/null @@ -1,233 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_gcc_generic_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_gcc_generic_H - -#include -#include - -#define __TBB_WORDSIZE __SIZEOF_POINTER__ - -#if __TBB_GCC_64BIT_ATOMIC_BUILTINS_BROKEN - #define __TBB_64BIT_ATOMICS 0 -#endif - -/** FPU control setting not available for non-Intel architectures on Android **/ -#if __ANDROID__ && __TBB_generic_arch - #define __TBB_CPU_CTL_ENV_PRESENT 0 -#endif - -// __BYTE_ORDER__ is used in accordance with http://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html, -// but __BIG_ENDIAN__ or __LITTLE_ENDIAN__ may be more commonly found instead. -#if __BIG_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_BIG -#elif __LITTLE_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE -#elif defined(__BYTE_ORDER__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_UNSUPPORTED -#else - #define __TBB_ENDIANNESS __TBB_ENDIAN_DETECT -#endif - -#if __TBB_GCC_VERSION < 40700 -// Use __sync_* builtins - -/** As this generic implementation has absolutely no information about underlying - hardware, its performance most likely will be sub-optimal because of full memory - fence usages where a more lightweight synchronization means (or none at all) - could suffice. Thus if you use this header to enable TBB on a new platform, - consider forking it and relaxing below helpers as appropriate. **/ -#define __TBB_acquire_consistency_helper() __sync_synchronize() -#define __TBB_release_consistency_helper() __sync_synchronize() -#define __TBB_full_memory_fence() __sync_synchronize() -#define __TBB_control_consistency_helper() __sync_synchronize() - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T) \ -inline T __TBB_machine_cmpswp##S( volatile void *ptr, T value, T comparand ) { \ - return __sync_val_compare_and_swap(reinterpret_cast(ptr),comparand,value); \ -} \ -inline T __TBB_machine_fetchadd##S( volatile void *ptr, T value ) { \ - return __sync_fetch_and_add(reinterpret_cast(ptr),value); \ -} - -#define __TBB_USE_GENERIC_FETCH_STORE 1 - -#else -// __TBB_GCC_VERSION >= 40700; use __atomic_* builtins available since gcc 4.7 - -#define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") -// Acquire and release fence intrinsics in GCC might miss compiler fence. -// Adding it at both sides of an intrinsic, as we do not know what reordering can be made. -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence(); __atomic_thread_fence(__ATOMIC_ACQUIRE); __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence(); __atomic_thread_fence(__ATOMIC_RELEASE); __TBB_compiler_fence() -#define __TBB_full_memory_fence() __atomic_thread_fence(__ATOMIC_SEQ_CST) -#define __TBB_control_consistency_helper() __TBB_acquire_consistency_helper() - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T) \ -inline T __TBB_machine_cmpswp##S( volatile void *ptr, T value, T comparand ) { \ - (void)__atomic_compare_exchange_n(reinterpret_cast(ptr), &comparand, value, \ - false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); \ - return comparand; \ -} \ -inline T __TBB_machine_fetchadd##S( volatile void *ptr, T value ) { \ - return __atomic_fetch_add(reinterpret_cast(ptr), value, __ATOMIC_SEQ_CST); \ -} \ -inline T __TBB_machine_fetchstore##S( volatile void *ptr, T value ) { \ - return __atomic_exchange_n(reinterpret_cast(ptr), value, __ATOMIC_SEQ_CST); \ -} - -#endif // __TBB_GCC_VERSION < 40700 - -__TBB_MACHINE_DEFINE_ATOMICS(1,int8_t) -__TBB_MACHINE_DEFINE_ATOMICS(2,int16_t) -__TBB_MACHINE_DEFINE_ATOMICS(4,int32_t) -__TBB_MACHINE_DEFINE_ATOMICS(8,int64_t) - -#undef __TBB_MACHINE_DEFINE_ATOMICS - -typedef unsigned char __TBB_Flag; -typedef __TBB_atomic __TBB_Flag __TBB_atomic_flag; - -#if __TBB_GCC_VERSION < 40700 -// Use __sync_* builtins - -// Use generic machine_load_store functions if there are no builtin atomics -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -static inline void __TBB_machine_or( volatile void *ptr, uintptr_t addend ) { - __sync_fetch_and_or(reinterpret_cast(ptr),addend); -} - -static inline void __TBB_machine_and( volatile void *ptr, uintptr_t addend ) { - __sync_fetch_and_and(reinterpret_cast(ptr),addend); -} - -inline bool __TBB_machine_try_lock_byte( __TBB_atomic_flag &flag ) { - return __sync_lock_test_and_set(&flag,1)==0; -} - -inline void __TBB_machine_unlock_byte( __TBB_atomic_flag &flag ) { - __sync_lock_release(&flag); -} - -#else -// __TBB_GCC_VERSION >= 40700; use __atomic_* builtins available since gcc 4.7 - -static inline void __TBB_machine_or( volatile void *ptr, uintptr_t addend ) { - __atomic_fetch_or(reinterpret_cast(ptr),addend,__ATOMIC_SEQ_CST); -} - -static inline void __TBB_machine_and( volatile void *ptr, uintptr_t addend ) { - __atomic_fetch_and(reinterpret_cast(ptr),addend,__ATOMIC_SEQ_CST); -} - -inline bool __TBB_machine_try_lock_byte( __TBB_atomic_flag &flag ) { - return !__atomic_test_and_set(&flag,__ATOMIC_ACQUIRE); -} - -inline void __TBB_machine_unlock_byte( __TBB_atomic_flag &flag ) { - __atomic_clear(&flag,__ATOMIC_RELEASE); -} - -namespace tbb { namespace internal { - -/** GCC atomic operation intrinsics might miss compiler fence. - Adding it after load-with-acquire, before store-with-release, and - on both sides of sequentially consistent operations is sufficient for correctness. **/ - -template -inline T __TBB_machine_atomic_load( const volatile T& location) { - if (MemOrder == __ATOMIC_SEQ_CST) __TBB_compiler_fence(); - T value = __atomic_load_n(&location, MemOrder); - if (MemOrder != __ATOMIC_RELAXED) __TBB_compiler_fence(); - return value; -} - -template -inline void __TBB_machine_atomic_store( volatile T& location, T value) { - if (MemOrder != __ATOMIC_RELAXED) __TBB_compiler_fence(); - __atomic_store_n(&location, value, MemOrder); - if (MemOrder == __ATOMIC_SEQ_CST) __TBB_compiler_fence(); -} - -template -struct machine_load_store { - static T load_with_acquire ( const volatile T& location ) { - return __TBB_machine_atomic_load(location); - } - static void store_with_release ( volatile T &location, T value ) { - __TBB_machine_atomic_store(location, value); - } -}; - -template -struct machine_load_store_relaxed { - static inline T load ( const volatile T& location ) { - return __TBB_machine_atomic_load(location); - } - static inline void store ( volatile T& location, T value ) { - __TBB_machine_atomic_store(location, value); - } -}; - -template -struct machine_load_store_seq_cst { - static T load ( const volatile T& location ) { - return __TBB_machine_atomic_load(location); - } - static void store ( volatile T &location, T value ) { - __TBB_machine_atomic_store(location, value); - } -}; - -}} // namespace tbb::internal - -#endif // __TBB_GCC_VERSION < 40700 - -// Machine specific atomic operations -#define __TBB_AtomicOR(P,V) __TBB_machine_or(P,V) -#define __TBB_AtomicAND(P,V) __TBB_machine_and(P,V) - -#define __TBB_TryLockByte __TBB_machine_try_lock_byte -#define __TBB_UnlockByte __TBB_machine_unlock_byte - -// __builtin_clz counts the number of leading zeroes -namespace tbb{ namespace internal { namespace gcc_builtins { - inline int clz(unsigned int x){ return __builtin_clz(x); } - inline int clz(unsigned long int x){ return __builtin_clzl(x); } - inline int clz(unsigned long long int x){ return __builtin_clzll(x); } -}}} -// logarithm is the index of the most significant non-zero bit -static inline intptr_t __TBB_machine_lg( uintptr_t x ) { - // If P is a power of 2 and x -static inline intptr_t __TBB_machine_lg( T x ) { - __TBB_ASSERT(x>0, "The logarithm of a non-positive value is undefined."); - uintptr_t j, i = x; - __asm__("bsr %1,%0" : "=r"(j) : "r"(i)); - return j; -} -#define __TBB_Log2(V) __TBB_machine_lg(V) -#endif /* !__TBB_Log2 */ - -#ifndef __TBB_Pause -//TODO: check if raising a ratio of pause instructions to loop control instructions -//(via e.g. loop unrolling) gives any benefit for HT. E.g, the current implementation -//does about 2 CPU-consuming instructions for every pause instruction. Perhaps for -//high pause counts it should use an unrolled loop to raise the ratio, and thus free -//up more integer cycles for the other hyperthread. On the other hand, if the loop is -//unrolled too far, it won't fit in the core's loop cache, and thus take away -//instruction decode slots from the other hyperthread. - -//TODO: check if use of gcc __builtin_ia32_pause intrinsic gives a "some how" better performing code -static inline void __TBB_machine_pause( int32_t delay ) { - for (int32_t i = 0; i < delay; i++) { - __asm__ __volatile__("pause;"); - } - return; -} -#define __TBB_Pause(V) __TBB_machine_pause(V) -#endif /* !__TBB_Pause */ - -namespace tbb { namespace internal { typedef uint64_t machine_tsc_t; } } -static inline tbb::internal::machine_tsc_t __TBB_machine_time_stamp() { -#if __INTEL_COMPILER - return _rdtsc(); -#else - tbb::internal::uint32_t hi, lo; - __asm__ __volatile__("rdtsc" : "=d"(hi), "=a"(lo)); - return (tbb::internal::machine_tsc_t( hi ) << 32) | lo; -#endif -} -#define __TBB_time_stamp() __TBB_machine_time_stamp() - -// API to retrieve/update FPU control setting -#ifndef __TBB_CPU_CTL_ENV_PRESENT -#define __TBB_CPU_CTL_ENV_PRESENT 1 -namespace tbb { -namespace internal { -class cpu_ctl_env { -private: - int mxcsr; - short x87cw; - static const int MXCSR_CONTROL_MASK = ~0x3f; /* all except last six status bits */ -public: - bool operator!=( const cpu_ctl_env& ctl ) const { return mxcsr != ctl.mxcsr || x87cw != ctl.x87cw; } - void get_env() { - #if __TBB_ICC_12_0_INL_ASM_FSTCW_BROKEN - cpu_ctl_env loc_ctl; - __asm__ __volatile__ ( - "stmxcsr %0\n\t" - "fstcw %1" - : "=m"(loc_ctl.mxcsr), "=m"(loc_ctl.x87cw) - ); - *this = loc_ctl; - #else - __asm__ __volatile__ ( - "stmxcsr %0\n\t" - "fstcw %1" - : "=m"(mxcsr), "=m"(x87cw) - ); - #endif - mxcsr &= MXCSR_CONTROL_MASK; - } - void set_env() const { - __asm__ __volatile__ ( - "ldmxcsr %0\n\t" - "fldcw %1" - : : "m"(mxcsr), "m"(x87cw) - ); - } -}; -} // namespace internal -} // namespace tbb -#endif /* !__TBB_CPU_CTL_ENV_PRESENT */ - -#include "gcc_itsx.h" - -#endif /* __TBB_machine_gcc_ia32_common_H */ diff --git a/src/3rd party/tbb/machine/gcc_itsx.h b/src/3rd party/tbb/machine/gcc_itsx.h deleted file mode 100644 index 5bd400e5cd..0000000000 --- a/src/3rd party/tbb/machine/gcc_itsx.h +++ /dev/null @@ -1,119 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_gcc_itsx_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_gcc_itsx_H - -#define __TBB_OP_XACQUIRE 0xF2 -#define __TBB_OP_XRELEASE 0xF3 -#define __TBB_OP_LOCK 0xF0 - -#define __TBB_STRINGIZE_INTERNAL(arg) #arg -#define __TBB_STRINGIZE(arg) __TBB_STRINGIZE_INTERNAL(arg) - -#ifdef __TBB_x86_64 -#define __TBB_r_out "=r" -#else -#define __TBB_r_out "=q" -#endif - -inline static uint8_t __TBB_machine_try_lock_elided( volatile uint8_t* lk ) -{ - uint8_t value = 1; - __asm__ volatile (".byte " __TBB_STRINGIZE(__TBB_OP_XACQUIRE)"; lock; xchgb %0, %1;" - : __TBB_r_out(value), "=m"(*lk) : "0"(value), "m"(*lk) : "memory" ); - return uint8_t(value^1); -} - -inline static void __TBB_machine_try_lock_elided_cancel() -{ - // 'pause' instruction aborts HLE/RTM transactions - __asm__ volatile ("pause\n" : : : "memory" ); -} - -inline static void __TBB_machine_unlock_elided( volatile uint8_t* lk ) -{ - __asm__ volatile (".byte " __TBB_STRINGIZE(__TBB_OP_XRELEASE)"; movb $0, %0" - : "=m"(*lk) : "m"(*lk) : "memory" ); -} - -#if __TBB_TSX_INTRINSICS_PRESENT -#include - -#define __TBB_machine_is_in_transaction _xtest -#define __TBB_machine_begin_transaction _xbegin -#define __TBB_machine_end_transaction _xend -#define __TBB_machine_transaction_conflict_abort() _xabort(0xff) - -#else - -/*! - * Check if the instruction is executed in a transaction or not - */ -inline static bool __TBB_machine_is_in_transaction() -{ - int8_t res = 0; -#if __TBB_x86_32 - __asm__ volatile (".byte 0x0F; .byte 0x01; .byte 0xD6;\n" - "setz %0" : "=q"(res) : : "memory" ); -#else - __asm__ volatile (".byte 0x0F; .byte 0x01; .byte 0xD6;\n" - "setz %0" : "=r"(res) : : "memory" ); -#endif - return res==0; -} - -/*! - * Enter speculative execution mode. - * @return -1 on success - * abort cause ( or 0 ) on abort - */ -inline static uint32_t __TBB_machine_begin_transaction() -{ - uint32_t res = ~uint32_t(0); // success value - __asm__ volatile ("1: .byte 0xC7; .byte 0xF8;\n" // XBEGIN - " .long 2f-1b-6\n" // 2f-1b == difference in addresses of start - // of XBEGIN and the MOVL - // 2f - 1b - 6 == that difference minus the size of the - // XBEGIN instruction. This is the abort offset to - // 2: below. - " jmp 3f\n" // success (leave -1 in res) - "2: movl %%eax,%0\n" // store failure code in res - "3:" - :"=r"(res):"0"(res):"memory","%eax"); - return res; -} - -/*! - * Attempt to commit/end transaction - */ -inline static void __TBB_machine_end_transaction() -{ - __asm__ volatile (".byte 0x0F; .byte 0x01; .byte 0xD5" :::"memory"); // XEND -} - -/* - * aborts with code 0xFF (lock already held) - */ -inline static void __TBB_machine_transaction_conflict_abort() -{ - __asm__ volatile (".byte 0xC6; .byte 0xF8; .byte 0xFF" :::"memory"); -} - -#endif /* __TBB_TSX_INTRINSICS_PRESENT */ diff --git a/src/3rd party/tbb/machine/ibm_aix51.h b/src/3rd party/tbb/machine/ibm_aix51.h deleted file mode 100644 index 14ba4d9fc9..0000000000 --- a/src/3rd party/tbb/machine/ibm_aix51.h +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -// TODO: revise by comparing with mac_ppc.h - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_ibm_aix51_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_ibm_aix51_H - -#define __TBB_WORDSIZE 8 -#define __TBB_ENDIANNESS __TBB_ENDIAN_BIG // assumption based on operating system - -#include -#include -#include - -extern "C" { -int32_t __TBB_machine_cas_32 (volatile void* ptr, int32_t value, int32_t comparand); -int64_t __TBB_machine_cas_64 (volatile void* ptr, int64_t value, int64_t comparand); -void __TBB_machine_flush (); -void __TBB_machine_lwsync (); -void __TBB_machine_isync (); -} - -// Mapping of old entry point names retained for the sake of backward binary compatibility -#define __TBB_machine_cmpswp4 __TBB_machine_cas_32 -#define __TBB_machine_cmpswp8 __TBB_machine_cas_64 - -#define __TBB_Yield() sched_yield() - -#define __TBB_USE_GENERIC_PART_WORD_CAS 1 -#define __TBB_USE_GENERIC_FETCH_ADD 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#if __GNUC__ - #define __TBB_control_consistency_helper() __asm__ __volatile__( "isync": : :"memory") - #define __TBB_acquire_consistency_helper() __asm__ __volatile__("lwsync": : :"memory") - #define __TBB_release_consistency_helper() __asm__ __volatile__("lwsync": : :"memory") - #define __TBB_full_memory_fence() __asm__ __volatile__( "sync": : :"memory") -#else - // IBM C++ Compiler does not support inline assembly - // TODO: Since XL 9.0 or earlier GCC syntax is supported. Replace with more - // lightweight implementation (like in mac_ppc.h) - #define __TBB_control_consistency_helper() __TBB_machine_isync () - #define __TBB_acquire_consistency_helper() __TBB_machine_lwsync () - #define __TBB_release_consistency_helper() __TBB_machine_lwsync () - #define __TBB_full_memory_fence() __TBB_machine_flush () -#endif diff --git a/src/3rd party/tbb/machine/icc_generic.h b/src/3rd party/tbb/machine/icc_generic.h deleted file mode 100644 index 00af78a782..0000000000 --- a/src/3rd party/tbb/machine/icc_generic.h +++ /dev/null @@ -1,258 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_icc_generic_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#if ! __TBB_ICC_BUILTIN_ATOMICS_PRESENT - #error "Intel(R) C++ Compiler of at least 12.0 version is needed to use ICC intrinsics port" -#endif - -#define __TBB_machine_icc_generic_H - -//ICC mimics the "native" target compiler -#if _MSC_VER - #include "msvc_ia32_common.h" -#else - #include "gcc_ia32_common.h" -#endif - -//TODO: Make __TBB_WORDSIZE macro optional for ICC intrinsics port. -//As compiler intrinsics are used for all the operations it is possible to do. - -#if __TBB_x86_32 - #define __TBB_WORDSIZE 4 -#else - #define __TBB_WORDSIZE 8 -#endif -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -//__TBB_compiler_fence() defined just in case, as it seems not to be used on its own anywhere else -#ifndef __TBB_compiler_fence -#if _MSC_VER - //TODO: any way to use same intrinsics on windows and linux? - #pragma intrinsic(_ReadWriteBarrier) - #define __TBB_compiler_fence() _ReadWriteBarrier() -#else - #define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") -#endif -#endif - -#ifndef __TBB_full_memory_fence -#if _MSC_VER - //TODO: any way to use same intrinsics on windows and linux? - #pragma intrinsic(_mm_mfence) - #define __TBB_full_memory_fence() _mm_mfence() -#else - #define __TBB_full_memory_fence() __asm__ __volatile__("mfence": : :"memory") -#endif -#endif - -#ifndef __TBB_control_consistency_helper -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#endif - -namespace tbb { namespace internal { -//TODO: is there any way to reuse definition of memory_order enum from ICC instead of copy paste. -//however it seems unlikely that ICC will silently change exact enum values, as they are defined -//in the ISO exactly like this. -//TODO: add test that exact values of the enum are same as in the ISO C++11 -typedef enum memory_order { - memory_order_relaxed, memory_order_consume, memory_order_acquire, - memory_order_release, memory_order_acq_rel, memory_order_seq_cst -} memory_order; - -namespace icc_intrinsics_port { - template - T convert_argument(T value){ - return value; - } - //The overload below is needed to have explicit conversion of pointer to void* in argument list. - //compiler bug? - //TODO: add according broken macro and recheck with ICC 13.0 if the overload is still needed - template - void* convert_argument(T* value){ - return (void*)value; - } -} -//TODO: code below is a bit repetitive, consider simplifying it -template -struct machine_load_store { - static T load_with_acquire ( const volatile T& location ) { - return __atomic_load_explicit(&location, memory_order_acquire); - } - static void store_with_release ( volatile T &location, T value ) { - __atomic_store_explicit(&location, icc_intrinsics_port::convert_argument(value), memory_order_release); - } -}; - -template -struct machine_load_store_relaxed { - static inline T load ( const T& location ) { - return __atomic_load_explicit(&location, memory_order_relaxed); - } - static inline void store ( T& location, T value ) { - __atomic_store_explicit(&location, icc_intrinsics_port::convert_argument(value), memory_order_relaxed); - } -}; - -template -struct machine_load_store_seq_cst { - static T load ( const volatile T& location ) { - return __atomic_load_explicit(&location, memory_order_seq_cst); - } - - static void store ( volatile T &location, T value ) { - __atomic_store_explicit(&location, value, memory_order_seq_cst); - } -}; - -}} // namespace tbb::internal - -namespace tbb{ namespace internal { namespace icc_intrinsics_port{ - typedef enum memory_order_map { - relaxed = memory_order_relaxed, - acquire = memory_order_acquire, - release = memory_order_release, - full_fence= memory_order_seq_cst - } memory_order_map; -}}}// namespace tbb::internal - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T,M) \ -inline T __TBB_machine_cmpswp##S##M( volatile void *ptr, T value, T comparand ) { \ - __atomic_compare_exchange_strong_explicit( \ - (T*)ptr \ - ,&comparand \ - ,value \ - , tbb::internal::icc_intrinsics_port::M \ - , tbb::internal::icc_intrinsics_port::M); \ - return comparand; \ -} \ - \ -inline T __TBB_machine_fetchstore##S##M(volatile void *ptr, T value) { \ - return __atomic_exchange_explicit((T*)ptr, value, tbb::internal::icc_intrinsics_port::M); \ -} \ - \ -inline T __TBB_machine_fetchadd##S##M(volatile void *ptr, T value) { \ - return __atomic_fetch_add_explicit((T*)ptr, value, tbb::internal::icc_intrinsics_port::M); \ -} \ - -__TBB_MACHINE_DEFINE_ATOMICS(1,tbb::internal::int8_t, full_fence) -__TBB_MACHINE_DEFINE_ATOMICS(1,tbb::internal::int8_t, acquire) -__TBB_MACHINE_DEFINE_ATOMICS(1,tbb::internal::int8_t, release) -__TBB_MACHINE_DEFINE_ATOMICS(1,tbb::internal::int8_t, relaxed) - -__TBB_MACHINE_DEFINE_ATOMICS(2,tbb::internal::int16_t, full_fence) -__TBB_MACHINE_DEFINE_ATOMICS(2,tbb::internal::int16_t, acquire) -__TBB_MACHINE_DEFINE_ATOMICS(2,tbb::internal::int16_t, release) -__TBB_MACHINE_DEFINE_ATOMICS(2,tbb::internal::int16_t, relaxed) - -__TBB_MACHINE_DEFINE_ATOMICS(4,tbb::internal::int32_t, full_fence) -__TBB_MACHINE_DEFINE_ATOMICS(4,tbb::internal::int32_t, acquire) -__TBB_MACHINE_DEFINE_ATOMICS(4,tbb::internal::int32_t, release) -__TBB_MACHINE_DEFINE_ATOMICS(4,tbb::internal::int32_t, relaxed) - -__TBB_MACHINE_DEFINE_ATOMICS(8,tbb::internal::int64_t, full_fence) -__TBB_MACHINE_DEFINE_ATOMICS(8,tbb::internal::int64_t, acquire) -__TBB_MACHINE_DEFINE_ATOMICS(8,tbb::internal::int64_t, release) -__TBB_MACHINE_DEFINE_ATOMICS(8,tbb::internal::int64_t, relaxed) - - -#undef __TBB_MACHINE_DEFINE_ATOMICS - -#define __TBB_USE_FENCED_ATOMICS 1 - -namespace tbb { namespace internal { -#if __TBB_FORCE_64BIT_ALIGNMENT_BROKEN -__TBB_MACHINE_DEFINE_LOAD8_GENERIC_FENCED(full_fence) -__TBB_MACHINE_DEFINE_STORE8_GENERIC_FENCED(full_fence) - -__TBB_MACHINE_DEFINE_LOAD8_GENERIC_FENCED(acquire) -__TBB_MACHINE_DEFINE_STORE8_GENERIC_FENCED(release) - -__TBB_MACHINE_DEFINE_LOAD8_GENERIC_FENCED(relaxed) -__TBB_MACHINE_DEFINE_STORE8_GENERIC_FENCED(relaxed) - -template -struct machine_load_store { - static T load_with_acquire ( const volatile T& location ) { - if( tbb::internal::is_aligned(&location,8)) { - return __atomic_load_explicit(&location, memory_order_acquire); - } else { - return __TBB_machine_generic_load8acquire(&location); - } - } - static void store_with_release ( volatile T &location, T value ) { - if( tbb::internal::is_aligned(&location,8)) { - __atomic_store_explicit(&location, icc_intrinsics_port::convert_argument(value), memory_order_release); - } else { - return __TBB_machine_generic_store8release(&location,value); - } - } -}; - -template -struct machine_load_store_relaxed { - static T load( const volatile T& location ) { - if( tbb::internal::is_aligned(&location,8)) { - return __atomic_load_explicit(&location, memory_order_relaxed); - } else { - return __TBB_machine_generic_load8relaxed(&location); - } - } - static void store( volatile T &location, T value ) { - if( tbb::internal::is_aligned(&location,8)) { - __atomic_store_explicit(&location, icc_intrinsics_port::convert_argument(value), memory_order_relaxed); - } else { - return __TBB_machine_generic_store8relaxed(&location,value); - } - } -}; - -template -struct machine_load_store_seq_cst { - static T load ( const volatile T& location ) { - if( tbb::internal::is_aligned(&location,8)) { - return __atomic_load_explicit(&location, memory_order_seq_cst); - } else { - return __TBB_machine_generic_load8full_fence(&location); - } - - } - - static void store ( volatile T &location, T value ) { - if( tbb::internal::is_aligned(&location,8)) { - __atomic_store_explicit(&location, value, memory_order_seq_cst); - } else { - return __TBB_machine_generic_store8full_fence(&location,value); - } - - } -}; - -#endif -}} // namespace tbb::internal -template -inline void __TBB_machine_OR( T *operand, T addend ) { - __atomic_fetch_or_explicit(operand, addend, tbb::internal::memory_order_seq_cst); -} - -template -inline void __TBB_machine_AND( T *operand, T addend ) { - __atomic_fetch_and_explicit(operand, addend, tbb::internal::memory_order_seq_cst); -} - diff --git a/src/3rd party/tbb/machine/linux_common.h b/src/3rd party/tbb/machine/linux_common.h deleted file mode 100644 index 6f93828d66..0000000000 --- a/src/3rd party/tbb/machine/linux_common.h +++ /dev/null @@ -1,105 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_machine_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#include -#define __TBB_Yield() sched_yield() - -#include -/* Futex definitions */ -#include - -#if defined(SYS_futex) -/* This header file is included for Linux and some other systems that may support futexes.*/ - -#define __TBB_USE_FUTEX 1 - -#if defined(__has_include) -#define __TBB_has_include __has_include -#else -#define __TBB_has_include(x) 0 -#endif - -/* -If available, use typical headers where futex API is defined. While Linux and OpenBSD -are known to provide such headers, other systems might have them as well. -*/ -#if defined(__linux__) || __TBB_has_include() -#include -#elif defined(__OpenBSD__) || __TBB_has_include() -#include -#endif - -#include -#include - -/* -Some systems might not define the macros or use different names. In such case we expect -the actual parameter values to match Linux: 0 for wait, 1 for wake. -*/ -#if defined(FUTEX_WAIT_PRIVATE) -#define __TBB_FUTEX_WAIT FUTEX_WAIT_PRIVATE -#elif defined(FUTEX_WAIT) -#define __TBB_FUTEX_WAIT FUTEX_WAIT -#else -#define __TBB_FUTEX_WAIT 0 -#endif - -#if defined(FUTEX_WAKE_PRIVATE) -#define __TBB_FUTEX_WAKE FUTEX_WAKE_PRIVATE -#elif defined(FUTEX_WAKE) -#define __TBB_FUTEX_WAKE FUTEX_WAKE -#else -#define __TBB_FUTEX_WAKE 1 -#endif - -#ifndef __TBB_ASSERT -#error machine specific headers must be included after tbb_stddef.h -#endif - -namespace tbb { - -namespace internal { - -inline int futex_wait( void *futex, int comparand ) { - int r = syscall( SYS_futex,futex,__TBB_FUTEX_WAIT,comparand,NULL,NULL,0 ); -#if TBB_USE_ASSERT - int e = errno; - __TBB_ASSERT( r==0||r==EWOULDBLOCK||(r==-1&&(e==EAGAIN||e==EINTR)), "futex_wait failed." ); -#endif /* TBB_USE_ASSERT */ - return r; -} - -inline int futex_wakeup_one( void *futex ) { - int r = ::syscall( SYS_futex,futex,__TBB_FUTEX_WAKE,1,NULL,NULL,0 ); - __TBB_ASSERT( r==0||r==1, "futex_wakeup_one: more than one thread woken up?" ); - return r; -} - -inline int futex_wakeup_all( void *futex ) { - int r = ::syscall( SYS_futex,futex,__TBB_FUTEX_WAKE,INT_MAX,NULL,NULL,0 ); - __TBB_ASSERT( r>=0, "futex_wakeup_all: error in waking up threads" ); - return r; -} - -} /* namespace internal */ - -} /* namespace tbb */ - -#endif /* SYS_futex */ diff --git a/src/3rd party/tbb/machine/linux_ia32.h b/src/3rd party/tbb/machine/linux_ia32.h deleted file mode 100644 index 6e2543c759..0000000000 --- a/src/3rd party/tbb/machine/linux_ia32.h +++ /dev/null @@ -1,228 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_linux_ia32_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_linux_ia32_H - -#include -#include "gcc_ia32_common.h" - -#define __TBB_WORDSIZE 4 -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -#define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence() -#define __TBB_full_memory_fence() __asm__ __volatile__("mfence": : :"memory") - -#if __TBB_ICC_ASM_VOLATILE_BROKEN -#define __TBB_VOLATILE -#else -#define __TBB_VOLATILE volatile -#endif - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T,X,R) \ -static inline T __TBB_machine_cmpswp##S (volatile void *ptr, T value, T comparand ) \ -{ \ - T result; \ - \ - __asm__ __volatile__("lock\ncmpxchg" X " %2,%1" \ - : "=a"(result), "=m"(*(__TBB_VOLATILE T*)ptr) \ - : "q"(value), "0"(comparand), "m"(*(__TBB_VOLATILE T*)ptr) \ - : "memory"); \ - return result; \ -} \ - \ -static inline T __TBB_machine_fetchadd##S(volatile void *ptr, T addend) \ -{ \ - T result; \ - __asm__ __volatile__("lock\nxadd" X " %0,%1" \ - : R (result), "=m"(*(__TBB_VOLATILE T*)ptr) \ - : "0"(addend), "m"(*(__TBB_VOLATILE T*)ptr) \ - : "memory"); \ - return result; \ -} \ - \ -static inline T __TBB_machine_fetchstore##S(volatile void *ptr, T value) \ -{ \ - T result; \ - __asm__ __volatile__("lock\nxchg" X " %0,%1" \ - : R (result), "=m"(*(__TBB_VOLATILE T*)ptr) \ - : "0"(value), "m"(*(__TBB_VOLATILE T*)ptr) \ - : "memory"); \ - return result; \ -} \ - -__TBB_MACHINE_DEFINE_ATOMICS(1,int8_t,"","=q") -__TBB_MACHINE_DEFINE_ATOMICS(2,int16_t,"","=r") -__TBB_MACHINE_DEFINE_ATOMICS(4,int32_t,"l","=r") - -#if __INTEL_COMPILER -#pragma warning( push ) -// reference to EBX in a function requiring stack alignment -#pragma warning( disable: 998 ) -#endif - -#if __TBB_GCC_CAS8_BUILTIN_INLINING_BROKEN -#define __TBB_IA32_CAS8_NOINLINE __attribute__ ((noinline)) -#else -#define __TBB_IA32_CAS8_NOINLINE -#endif - -static inline __TBB_IA32_CAS8_NOINLINE int64_t __TBB_machine_cmpswp8 (volatile void *ptr, int64_t value, int64_t comparand ) { -//TODO: remove the extra part of condition once __TBB_GCC_BUILTIN_ATOMICS_PRESENT is lowered to gcc version 4.1.2 -#if (__TBB_GCC_BUILTIN_ATOMICS_PRESENT || (__TBB_GCC_VERSION >= 40102)) && !__TBB_GCC_64BIT_ATOMIC_BUILTINS_BROKEN - return __sync_val_compare_and_swap( reinterpret_cast(ptr), comparand, value ); -#else /* !__TBB_GCC_BUILTIN_ATOMICS_PRESENT */ - //TODO: look like ICC 13.0 has some issues with this code, investigate it more deeply - int64_t result; - union { - int64_t i64; - int32_t i32[2]; - }; - i64 = value; -#if __PIC__ - /* compiling position-independent code */ - // EBX register preserved for compliance with position-independent code rules on IA32 - int32_t tmp; - __asm__ __volatile__ ( - "movl %%ebx,%2\n\t" - "movl %5,%%ebx\n\t" -#if __GNUC__==3 - "lock\n\t cmpxchg8b %1\n\t" -#else - "lock\n\t cmpxchg8b (%3)\n\t" -#endif - "movl %2,%%ebx" - : "=A"(result) - , "=m"(*(__TBB_VOLATILE int64_t *)ptr) - , "=m"(tmp) -#if __GNUC__==3 - : "m"(*(__TBB_VOLATILE int64_t *)ptr) -#else - : "SD"(ptr) -#endif - , "0"(comparand) - , "m"(i32[0]), "c"(i32[1]) - : "memory" -#if __INTEL_COMPILER - ,"ebx" -#endif - ); -#else /* !__PIC__ */ - __asm__ __volatile__ ( - "lock\n\t cmpxchg8b %1\n\t" - : "=A"(result), "=m"(*(__TBB_VOLATILE int64_t *)ptr) - : "m"(*(__TBB_VOLATILE int64_t *)ptr) - , "0"(comparand) - , "b"(i32[0]), "c"(i32[1]) - : "memory" - ); -#endif /* __PIC__ */ - return result; -#endif /* !__TBB_GCC_BUILTIN_ATOMICS_PRESENT */ -} - -#undef __TBB_IA32_CAS8_NOINLINE - -#if __INTEL_COMPILER -#pragma warning( pop ) -#endif // warning 998 is back - -static inline void __TBB_machine_or( volatile void *ptr, uint32_t addend ) { - __asm__ __volatile__("lock\norl %1,%0" : "=m"(*(__TBB_VOLATILE uint32_t *)ptr) : "r"(addend), "m"(*(__TBB_VOLATILE uint32_t *)ptr) : "memory"); -} - -static inline void __TBB_machine_and( volatile void *ptr, uint32_t addend ) { - __asm__ __volatile__("lock\nandl %1,%0" : "=m"(*(__TBB_VOLATILE uint32_t *)ptr) : "r"(addend), "m"(*(__TBB_VOLATILE uint32_t *)ptr) : "memory"); -} - -//TODO: Check if it possible and profitable for IA-32 architecture on (Linux* and Windows*) -//to use of 64-bit load/store via floating point registers together with full fence -//for sequentially consistent load/store, instead of CAS. - -#if __clang__ -#define __TBB_fildq "fildll" -#define __TBB_fistpq "fistpll" -#else -#define __TBB_fildq "fildq" -#define __TBB_fistpq "fistpq" -#endif - -static inline int64_t __TBB_machine_aligned_load8 (const volatile void *ptr) { - __TBB_ASSERT(tbb::internal::is_aligned(ptr,8),"__TBB_machine_aligned_load8 should be used with 8 byte aligned locations only \n"); - int64_t result; - __asm__ __volatile__ ( __TBB_fildq " %1\n\t" - __TBB_fistpq " %0" : "=m"(result) : "m"(*(const __TBB_VOLATILE uint64_t*)ptr) : "memory" ); - return result; -} - -static inline void __TBB_machine_aligned_store8 (volatile void *ptr, int64_t value ) { - __TBB_ASSERT(tbb::internal::is_aligned(ptr,8),"__TBB_machine_aligned_store8 should be used with 8 byte aligned locations only \n"); - // Aligned store - __asm__ __volatile__ ( __TBB_fildq " %1\n\t" - __TBB_fistpq " %0" : "=m"(*(__TBB_VOLATILE int64_t*)ptr) : "m"(value) : "memory" ); -} - -static inline int64_t __TBB_machine_load8 (const volatile void *ptr) { -#if __TBB_FORCE_64BIT_ALIGNMENT_BROKEN - if( tbb::internal::is_aligned(ptr,8)) { -#endif - return __TBB_machine_aligned_load8(ptr); -#if __TBB_FORCE_64BIT_ALIGNMENT_BROKEN - } else { - // Unaligned load - return __TBB_machine_cmpswp8(const_cast(ptr),0,0); - } -#endif -} - -//! Handles misaligned 8-byte store -/** Defined in tbb_misc.cpp */ -extern "C" void __TBB_machine_store8_slow( volatile void *ptr, int64_t value ); -extern "C" void __TBB_machine_store8_slow_perf_warning( volatile void *ptr ); - -static inline void __TBB_machine_store8(volatile void *ptr, int64_t value) { -#if __TBB_FORCE_64BIT_ALIGNMENT_BROKEN - if( tbb::internal::is_aligned(ptr,8)) { -#endif - __TBB_machine_aligned_store8(ptr,value); -#if __TBB_FORCE_64BIT_ALIGNMENT_BROKEN - } else { - // Unaligned store -#if TBB_USE_PERFORMANCE_WARNINGS - __TBB_machine_store8_slow_perf_warning(ptr); -#endif /* TBB_USE_PERFORMANCE_WARNINGS */ - __TBB_machine_store8_slow(ptr,value); - } -#endif -} - -// Machine specific atomic operations -#define __TBB_AtomicOR(P,V) __TBB_machine_or(P,V) -#define __TBB_AtomicAND(P,V) __TBB_machine_and(P,V) - -#define __TBB_USE_GENERIC_DWORD_FETCH_ADD 1 -#define __TBB_USE_GENERIC_DWORD_FETCH_STORE 1 -#define __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - diff --git a/src/3rd party/tbb/machine/linux_ia64.h b/src/3rd party/tbb/machine/linux_ia64.h deleted file mode 100644 index be9676e1ca..0000000000 --- a/src/3rd party/tbb/machine/linux_ia64.h +++ /dev/null @@ -1,177 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_linux_ia64_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_linux_ia64_H - -#include -#include - -#define __TBB_WORDSIZE 8 -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -#if __INTEL_COMPILER - #define __TBB_compiler_fence() - #define __TBB_control_consistency_helper() __TBB_compiler_fence() - #define __TBB_acquire_consistency_helper() - #define __TBB_release_consistency_helper() - #define __TBB_full_memory_fence() __mf() -#else - #define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") - #define __TBB_control_consistency_helper() __TBB_compiler_fence() - // Even though GCC imbues volatile loads with acquire semantics, it sometimes moves - // loads over the acquire fence. The following helpers stop such incorrect code motion. - #define __TBB_acquire_consistency_helper() __TBB_compiler_fence() - #define __TBB_release_consistency_helper() __TBB_compiler_fence() - #define __TBB_full_memory_fence() __asm__ __volatile__("mf": : :"memory") -#endif /* !__INTEL_COMPILER */ - -// Most of the functions will be in a .s file -// TODO: revise dynamic_link, memory pools and etc. if the library dependency is removed. - -extern "C" { - int8_t __TBB_machine_fetchadd1__TBB_full_fence (volatile void *ptr, int8_t addend); - int8_t __TBB_machine_fetchadd1acquire(volatile void *ptr, int8_t addend); - int8_t __TBB_machine_fetchadd1release(volatile void *ptr, int8_t addend); - - int16_t __TBB_machine_fetchadd2__TBB_full_fence (volatile void *ptr, int16_t addend); - int16_t __TBB_machine_fetchadd2acquire(volatile void *ptr, int16_t addend); - int16_t __TBB_machine_fetchadd2release(volatile void *ptr, int16_t addend); - - int32_t __TBB_machine_fetchadd4__TBB_full_fence (volatile void *ptr, int32_t value); - int32_t __TBB_machine_fetchadd4acquire(volatile void *ptr, int32_t addend); - int32_t __TBB_machine_fetchadd4release(volatile void *ptr, int32_t addend); - - int64_t __TBB_machine_fetchadd8__TBB_full_fence (volatile void *ptr, int64_t value); - int64_t __TBB_machine_fetchadd8acquire(volatile void *ptr, int64_t addend); - int64_t __TBB_machine_fetchadd8release(volatile void *ptr, int64_t addend); - - int8_t __TBB_machine_fetchstore1__TBB_full_fence (volatile void *ptr, int8_t value); - int8_t __TBB_machine_fetchstore1acquire(volatile void *ptr, int8_t value); - int8_t __TBB_machine_fetchstore1release(volatile void *ptr, int8_t value); - - int16_t __TBB_machine_fetchstore2__TBB_full_fence (volatile void *ptr, int16_t value); - int16_t __TBB_machine_fetchstore2acquire(volatile void *ptr, int16_t value); - int16_t __TBB_machine_fetchstore2release(volatile void *ptr, int16_t value); - - int32_t __TBB_machine_fetchstore4__TBB_full_fence (volatile void *ptr, int32_t value); - int32_t __TBB_machine_fetchstore4acquire(volatile void *ptr, int32_t value); - int32_t __TBB_machine_fetchstore4release(volatile void *ptr, int32_t value); - - int64_t __TBB_machine_fetchstore8__TBB_full_fence (volatile void *ptr, int64_t value); - int64_t __TBB_machine_fetchstore8acquire(volatile void *ptr, int64_t value); - int64_t __TBB_machine_fetchstore8release(volatile void *ptr, int64_t value); - - int8_t __TBB_machine_cmpswp1__TBB_full_fence (volatile void *ptr, int8_t value, int8_t comparand); - int8_t __TBB_machine_cmpswp1acquire(volatile void *ptr, int8_t value, int8_t comparand); - int8_t __TBB_machine_cmpswp1release(volatile void *ptr, int8_t value, int8_t comparand); - - int16_t __TBB_machine_cmpswp2__TBB_full_fence (volatile void *ptr, int16_t value, int16_t comparand); - int16_t __TBB_machine_cmpswp2acquire(volatile void *ptr, int16_t value, int16_t comparand); - int16_t __TBB_machine_cmpswp2release(volatile void *ptr, int16_t value, int16_t comparand); - - int32_t __TBB_machine_cmpswp4__TBB_full_fence (volatile void *ptr, int32_t value, int32_t comparand); - int32_t __TBB_machine_cmpswp4acquire(volatile void *ptr, int32_t value, int32_t comparand); - int32_t __TBB_machine_cmpswp4release(volatile void *ptr, int32_t value, int32_t comparand); - - int64_t __TBB_machine_cmpswp8__TBB_full_fence (volatile void *ptr, int64_t value, int64_t comparand); - int64_t __TBB_machine_cmpswp8acquire(volatile void *ptr, int64_t value, int64_t comparand); - int64_t __TBB_machine_cmpswp8release(volatile void *ptr, int64_t value, int64_t comparand); - - int64_t __TBB_machine_lg(uint64_t value); - void __TBB_machine_pause(int32_t delay); - bool __TBB_machine_trylockbyte( volatile unsigned char &ptr ); - int64_t __TBB_machine_lockbyte( volatile unsigned char &ptr ); - - //! Retrieves the current RSE backing store pointer. IA64 specific. - void* __TBB_get_bsp(); - - int32_t __TBB_machine_load1_relaxed(const void *ptr); - int32_t __TBB_machine_load2_relaxed(const void *ptr); - int32_t __TBB_machine_load4_relaxed(const void *ptr); - int64_t __TBB_machine_load8_relaxed(const void *ptr); - - void __TBB_machine_store1_relaxed(void *ptr, int32_t value); - void __TBB_machine_store2_relaxed(void *ptr, int32_t value); - void __TBB_machine_store4_relaxed(void *ptr, int32_t value); - void __TBB_machine_store8_relaxed(void *ptr, int64_t value); -} // extern "C" - -// Mapping old entry points to the names corresponding to the new full_fence identifier. -#define __TBB_machine_fetchadd1full_fence __TBB_machine_fetchadd1__TBB_full_fence -#define __TBB_machine_fetchadd2full_fence __TBB_machine_fetchadd2__TBB_full_fence -#define __TBB_machine_fetchadd4full_fence __TBB_machine_fetchadd4__TBB_full_fence -#define __TBB_machine_fetchadd8full_fence __TBB_machine_fetchadd8__TBB_full_fence -#define __TBB_machine_fetchstore1full_fence __TBB_machine_fetchstore1__TBB_full_fence -#define __TBB_machine_fetchstore2full_fence __TBB_machine_fetchstore2__TBB_full_fence -#define __TBB_machine_fetchstore4full_fence __TBB_machine_fetchstore4__TBB_full_fence -#define __TBB_machine_fetchstore8full_fence __TBB_machine_fetchstore8__TBB_full_fence -#define __TBB_machine_cmpswp1full_fence __TBB_machine_cmpswp1__TBB_full_fence -#define __TBB_machine_cmpswp2full_fence __TBB_machine_cmpswp2__TBB_full_fence -#define __TBB_machine_cmpswp4full_fence __TBB_machine_cmpswp4__TBB_full_fence -#define __TBB_machine_cmpswp8full_fence __TBB_machine_cmpswp8__TBB_full_fence - -// Mapping relaxed operations to the entry points implementing them. -/** On IA64 RMW operations implicitly have acquire semantics. Thus one cannot - actually have completely relaxed RMW operation here. **/ -#define __TBB_machine_fetchadd1relaxed __TBB_machine_fetchadd1acquire -#define __TBB_machine_fetchadd2relaxed __TBB_machine_fetchadd2acquire -#define __TBB_machine_fetchadd4relaxed __TBB_machine_fetchadd4acquire -#define __TBB_machine_fetchadd8relaxed __TBB_machine_fetchadd8acquire -#define __TBB_machine_fetchstore1relaxed __TBB_machine_fetchstore1acquire -#define __TBB_machine_fetchstore2relaxed __TBB_machine_fetchstore2acquire -#define __TBB_machine_fetchstore4relaxed __TBB_machine_fetchstore4acquire -#define __TBB_machine_fetchstore8relaxed __TBB_machine_fetchstore8acquire -#define __TBB_machine_cmpswp1relaxed __TBB_machine_cmpswp1acquire -#define __TBB_machine_cmpswp2relaxed __TBB_machine_cmpswp2acquire -#define __TBB_machine_cmpswp4relaxed __TBB_machine_cmpswp4acquire -#define __TBB_machine_cmpswp8relaxed __TBB_machine_cmpswp8acquire - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,V) \ - template \ - struct machine_load_store_relaxed { \ - static inline T load ( const T& location ) { \ - return (T)__TBB_machine_load##S##_relaxed(&location); \ - } \ - static inline void store ( T& location, T value ) { \ - __TBB_machine_store##S##_relaxed(&location, (V)value); \ - } \ - } - -namespace tbb { -namespace internal { - __TBB_MACHINE_DEFINE_ATOMICS(1,int8_t); - __TBB_MACHINE_DEFINE_ATOMICS(2,int16_t); - __TBB_MACHINE_DEFINE_ATOMICS(4,int32_t); - __TBB_MACHINE_DEFINE_ATOMICS(8,int64_t); -}} // namespaces internal, tbb - -#undef __TBB_MACHINE_DEFINE_ATOMICS - -#define __TBB_USE_FENCED_ATOMICS 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -// Definition of Lock functions -#define __TBB_TryLockByte(P) __TBB_machine_trylockbyte(P) -#define __TBB_LockByte(P) __TBB_machine_lockbyte(P) - -// Definition of other utility functions -#define __TBB_Pause(V) __TBB_machine_pause(V) -#define __TBB_Log2(V) __TBB_machine_lg(V) diff --git a/src/3rd party/tbb/machine/linux_intel64.h b/src/3rd party/tbb/machine/linux_intel64.h deleted file mode 100644 index d0b15bed18..0000000000 --- a/src/3rd party/tbb/machine/linux_intel64.h +++ /dev/null @@ -1,92 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_linux_intel64_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_linux_intel64_H - -#include -#include "gcc_ia32_common.h" - -#define __TBB_WORDSIZE 8 -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -#define __TBB_compiler_fence() __asm__ __volatile__("": : :"memory") -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence() - -#ifndef __TBB_full_memory_fence -#define __TBB_full_memory_fence() __asm__ __volatile__("mfence": : :"memory") -#endif - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T,X) \ -static inline T __TBB_machine_cmpswp##S (volatile void *ptr, T value, T comparand ) \ -{ \ - T result; \ - \ - __asm__ __volatile__("lock\ncmpxchg" X " %2,%1" \ - : "=a"(result), "=m"(*(volatile T*)ptr) \ - : "q"(value), "0"(comparand), "m"(*(volatile T*)ptr) \ - : "memory"); \ - return result; \ -} \ - \ -static inline T __TBB_machine_fetchadd##S(volatile void *ptr, T addend) \ -{ \ - T result; \ - __asm__ __volatile__("lock\nxadd" X " %0,%1" \ - : "=r"(result),"=m"(*(volatile T*)ptr) \ - : "0"(addend), "m"(*(volatile T*)ptr) \ - : "memory"); \ - return result; \ -} \ - \ -static inline T __TBB_machine_fetchstore##S(volatile void *ptr, T value) \ -{ \ - T result; \ - __asm__ __volatile__("lock\nxchg" X " %0,%1" \ - : "=r"(result),"=m"(*(volatile T*)ptr) \ - : "0"(value), "m"(*(volatile T*)ptr) \ - : "memory"); \ - return result; \ -} \ - -__TBB_MACHINE_DEFINE_ATOMICS(1,int8_t,"") -__TBB_MACHINE_DEFINE_ATOMICS(2,int16_t,"") -__TBB_MACHINE_DEFINE_ATOMICS(4,int32_t,"") -__TBB_MACHINE_DEFINE_ATOMICS(8,int64_t,"q") - -#undef __TBB_MACHINE_DEFINE_ATOMICS - -static inline void __TBB_machine_or( volatile void *ptr, uint64_t value ) { - __asm__ __volatile__("lock\norq %1,%0" : "=m"(*(volatile uint64_t*)ptr) : "r"(value), "m"(*(volatile uint64_t*)ptr) : "memory"); -} - -static inline void __TBB_machine_and( volatile void *ptr, uint64_t value ) { - __asm__ __volatile__("lock\nandq %1,%0" : "=m"(*(volatile uint64_t*)ptr) : "r"(value), "m"(*(volatile uint64_t*)ptr) : "memory"); -} - -#define __TBB_AtomicOR(P,V) __TBB_machine_or(P,V) -#define __TBB_AtomicAND(P,V) __TBB_machine_and(P,V) - -#define __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - diff --git a/src/3rd party/tbb/machine/mac_ppc.h b/src/3rd party/tbb/machine/mac_ppc.h deleted file mode 100644 index 0c07cc99f2..0000000000 --- a/src/3rd party/tbb/machine/mac_ppc.h +++ /dev/null @@ -1,309 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_gcc_power_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_gcc_power_H - -#include -#include - -// TODO: rename to gcc_power.h? -// This file is for Power Architecture with compilers supporting GNU inline-assembler syntax (currently GNU g++ and IBM XL). -// Note that XL V9.0 (sometimes?) has trouble dealing with empty input and/or clobber lists, so they should be avoided. - -#if __powerpc64__ || __ppc64__ - // IBM XL documents __powerpc64__ (and __PPC64__). - // Apple documents __ppc64__ (with __ppc__ only on 32-bit). - #define __TBB_WORDSIZE 8 -#else - #define __TBB_WORDSIZE 4 -#endif - -// Traditionally Power Architecture is big-endian. -// Little-endian could be just an address manipulation (compatibility with TBB not verified), -// or normal little-endian (on more recent systems). Embedded PowerPC systems may support -// page-specific endianness, but then one endianness must be hidden from TBB so that it still sees only one. -#if __BIG_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_BIG -#elif __LITTLE_ENDIAN__ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE -#elif defined(__BYTE_ORDER__) - #define __TBB_ENDIANNESS __TBB_ENDIAN_UNSUPPORTED -#else - #define __TBB_ENDIANNESS __TBB_ENDIAN_DETECT -#endif - -// On Power Architecture, (lock-free) 64-bit atomics require 64-bit hardware: -#if __TBB_WORDSIZE==8 - // Do not change the following definition, because TBB itself will use 64-bit atomics in 64-bit builds. - #define __TBB_64BIT_ATOMICS 1 -#elif __bgp__ - // Do not change the following definition, because this is known 32-bit hardware. - #define __TBB_64BIT_ATOMICS 0 -#else - // To enable 64-bit atomics in 32-bit builds, set the value below to 1 instead of 0. - // You must make certain that the program will only use them on actual 64-bit hardware - // (which typically means that the entire program is only executed on such hardware), - // because their implementation involves machine instructions that are illegal elsewhere. - // The setting can be chosen independently per compilation unit, - // which also means that TBB itself does not need to be rebuilt. - // Alternatively (but only for the current architecture and TBB version), - // override the default as a predefined macro when invoking the compiler. - #ifndef __TBB_64BIT_ATOMICS - #define __TBB_64BIT_ATOMICS 0 - #endif -#endif - -inline int32_t __TBB_machine_cmpswp4 (volatile void *ptr, int32_t value, int32_t comparand ) -{ - int32_t result; - - __asm__ __volatile__("sync\n" - "0:\n\t" - "lwarx %[res],0,%[ptr]\n\t" /* load w/ reservation */ - "cmpw %[res],%[cmp]\n\t" /* compare against comparand */ - "bne- 1f\n\t" /* exit if not same */ - "stwcx. %[val],0,%[ptr]\n\t" /* store new value */ - "bne- 0b\n" /* retry if reservation lost */ - "1:\n\t" /* the exit */ - "isync" - : [res]"=&r"(result) - , "+m"(* (int32_t*) ptr) /* redundant with "memory" */ - : [ptr]"r"(ptr) - , [val]"r"(value) - , [cmp]"r"(comparand) - : "memory" /* compiler full fence */ - , "cr0" /* clobbered by cmp and/or stwcx. */ - ); - return result; -} - -#if __TBB_WORDSIZE==8 - -inline int64_t __TBB_machine_cmpswp8 (volatile void *ptr, int64_t value, int64_t comparand ) -{ - int64_t result; - __asm__ __volatile__("sync\n" - "0:\n\t" - "ldarx %[res],0,%[ptr]\n\t" /* load w/ reservation */ - "cmpd %[res],%[cmp]\n\t" /* compare against comparand */ - "bne- 1f\n\t" /* exit if not same */ - "stdcx. %[val],0,%[ptr]\n\t" /* store new value */ - "bne- 0b\n" /* retry if reservation lost */ - "1:\n\t" /* the exit */ - "isync" - : [res]"=&r"(result) - , "+m"(* (int64_t*) ptr) /* redundant with "memory" */ - : [ptr]"r"(ptr) - , [val]"r"(value) - , [cmp]"r"(comparand) - : "memory" /* compiler full fence */ - , "cr0" /* clobbered by cmp and/or stdcx. */ - ); - return result; -} - -#elif __TBB_64BIT_ATOMICS /* && __TBB_WORDSIZE==4 */ - -inline int64_t __TBB_machine_cmpswp8 (volatile void *ptr, int64_t value, int64_t comparand ) -{ - int64_t result; - int64_t value_register, comparand_register, result_register; // dummy variables to allocate registers - __asm__ __volatile__("sync\n\t" - "ld %[val],%[valm]\n\t" - "ld %[cmp],%[cmpm]\n" - "0:\n\t" - "ldarx %[res],0,%[ptr]\n\t" /* load w/ reservation */ - "cmpd %[res],%[cmp]\n\t" /* compare against comparand */ - "bne- 1f\n\t" /* exit if not same */ - "stdcx. %[val],0,%[ptr]\n\t" /* store new value */ - "bne- 0b\n" /* retry if reservation lost */ - "1:\n\t" /* the exit */ - "std %[res],%[resm]\n\t" - "isync" - : [resm]"=m"(result) - , [res] "=&r"( result_register) - , [val] "=&r"( value_register) - , [cmp] "=&r"(comparand_register) - , "+m"(* (int64_t*) ptr) /* redundant with "memory" */ - : [ptr] "r"(ptr) - , [valm]"m"(value) - , [cmpm]"m"(comparand) - : "memory" /* compiler full fence */ - , "cr0" /* clobbered by cmpd and/or stdcx. */ - ); - return result; -} - -#endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */ - -#define __TBB_MACHINE_DEFINE_LOAD_STORE(S,ldx,stx,cmpx) \ - template \ - struct machine_load_store { \ - static inline T load_with_acquire(const volatile T& location) { \ - T result; \ - __asm__ __volatile__(ldx " %[res],0(%[ptr])\n" \ - "0:\n\t" \ - cmpx " %[res],%[res]\n\t" \ - "bne- 0b\n\t" \ - "isync" \ - : [res]"=r"(result) \ - : [ptr]"b"(&location) /* cannot use register 0 here */ \ - , "m"(location) /* redundant with "memory" */ \ - : "memory" /* compiler acquire fence */ \ - , "cr0" /* clobbered by cmpw/cmpd */); \ - return result; \ - } \ - static inline void store_with_release(volatile T &location, T value) { \ - __asm__ __volatile__("lwsync\n\t" \ - stx " %[val],0(%[ptr])" \ - : "=m"(location) /* redundant with "memory" */ \ - : [ptr]"b"(&location) /* cannot use register 0 here */ \ - , [val]"r"(value) \ - : "memory"/*compiler release fence*/ /*(cr0 not affected)*/); \ - } \ - }; \ - \ - template \ - struct machine_load_store_relaxed { \ - static inline T load (const __TBB_atomic T& location) { \ - T result; \ - __asm__ __volatile__(ldx " %[res],0(%[ptr])" \ - : [res]"=r"(result) \ - : [ptr]"b"(&location) /* cannot use register 0 here */ \ - , "m"(location) \ - ); /*(no compiler fence)*/ /*(cr0 not affected)*/ \ - return result; \ - } \ - static inline void store (__TBB_atomic T &location, T value) { \ - __asm__ __volatile__(stx " %[val],0(%[ptr])" \ - : "=m"(location) \ - : [ptr]"b"(&location) /* cannot use register 0 here */ \ - , [val]"r"(value) \ - ); /*(no compiler fence)*/ /*(cr0 not affected)*/ \ - } \ - }; - -namespace tbb { -namespace internal { - __TBB_MACHINE_DEFINE_LOAD_STORE(1,"lbz","stb","cmpw") - __TBB_MACHINE_DEFINE_LOAD_STORE(2,"lhz","sth","cmpw") - __TBB_MACHINE_DEFINE_LOAD_STORE(4,"lwz","stw","cmpw") - -#if __TBB_WORDSIZE==8 - - __TBB_MACHINE_DEFINE_LOAD_STORE(8,"ld" ,"std","cmpd") - -#elif __TBB_64BIT_ATOMICS /* && __TBB_WORDSIZE==4 */ - - template - struct machine_load_store { - static inline T load_with_acquire(const volatile T& location) { - T result; - T result_register; // dummy variable to allocate a register - __asm__ __volatile__("ld %[res],0(%[ptr])\n\t" - "std %[res],%[resm]\n" - "0:\n\t" - "cmpd %[res],%[res]\n\t" - "bne- 0b\n\t" - "isync" - : [resm]"=m"(result) - , [res]"=&r"(result_register) - : [ptr]"b"(&location) /* cannot use register 0 here */ - , "m"(location) /* redundant with "memory" */ - : "memory" /* compiler acquire fence */ - , "cr0" /* clobbered by cmpd */); - return result; - } - - static inline void store_with_release(volatile T &location, T value) { - T value_register; // dummy variable to allocate a register - __asm__ __volatile__("lwsync\n\t" - "ld %[val],%[valm]\n\t" - "std %[val],0(%[ptr])" - : "=m"(location) /* redundant with "memory" */ - , [val]"=&r"(value_register) - : [ptr]"b"(&location) /* cannot use register 0 here */ - , [valm]"m"(value) - : "memory"/*compiler release fence*/ /*(cr0 not affected)*/); - } - }; - - struct machine_load_store_relaxed { - static inline T load (const volatile T& location) { - T result; - T result_register; // dummy variable to allocate a register - __asm__ __volatile__("ld %[res],0(%[ptr])\n\t" - "std %[res],%[resm]" - : [resm]"=m"(result) - , [res]"=&r"(result_register) - : [ptr]"b"(&location) /* cannot use register 0 here */ - , "m"(location) - ); /*(no compiler fence)*/ /*(cr0 not affected)*/ - return result; - } - - static inline void store (volatile T &location, T value) { - T value_register; // dummy variable to allocate a register - __asm__ __volatile__("ld %[val],%[valm]\n\t" - "std %[val],0(%[ptr])" - : "=m"(location) - , [val]"=&r"(value_register) - : [ptr]"b"(&location) /* cannot use register 0 here */ - , [valm]"m"(value) - ); /*(no compiler fence)*/ /*(cr0 not affected)*/ - } - }; - #define __TBB_machine_load_store_relaxed_8 - -#endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */ - -}} // namespaces internal, tbb - -#undef __TBB_MACHINE_DEFINE_LOAD_STORE - -#define __TBB_USE_GENERIC_PART_WORD_CAS 1 -#define __TBB_USE_GENERIC_FETCH_ADD 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#define __TBB_control_consistency_helper() __asm__ __volatile__("isync": : :"memory") -#define __TBB_full_memory_fence() __asm__ __volatile__( "sync": : :"memory") - -static inline intptr_t __TBB_machine_lg( uintptr_t x ) { - __TBB_ASSERT(x, "__TBB_Log2(0) undefined"); - // cntlzd/cntlzw starts counting at 2^63/2^31 (ignoring any higher-order bits), and does not affect cr0 -#if __TBB_WORDSIZE==8 - __asm__ __volatile__ ("cntlzd %0,%0" : "+r"(x)); - return 63-static_cast(x); -#else - __asm__ __volatile__ ("cntlzw %0,%0" : "+r"(x)); - return 31-static_cast(x); -#endif -} -#define __TBB_Log2(V) __TBB_machine_lg(V) - -// Assumes implicit alignment for any 32-bit value -typedef uint32_t __TBB_Flag; -#define __TBB_Flag __TBB_Flag - -inline bool __TBB_machine_trylockbyte( __TBB_atomic __TBB_Flag &flag ) { - return __TBB_machine_cmpswp4(&flag,1,0)==0; -} -#define __TBB_TryLockByte(P) __TBB_machine_trylockbyte(P) diff --git a/src/3rd party/tbb/machine/macos_common.h b/src/3rd party/tbb/machine/macos_common.h deleted file mode 100644 index c63536fae5..0000000000 --- a/src/3rd party/tbb/machine/macos_common.h +++ /dev/null @@ -1,129 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_macos_common_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_macos_common_H - -#include -#define __TBB_Yield() sched_yield() - -// __TBB_HardwareConcurrency - -#include -#include - -static inline int __TBB_macos_available_cpu() { - int name[2] = {CTL_HW, HW_AVAILCPU}; - int ncpu; - size_t size = sizeof(ncpu); - sysctl( name, 2, &ncpu, &size, NULL, 0 ); - return ncpu; -} - -#define __TBB_HardwareConcurrency() __TBB_macos_available_cpu() - -#ifndef __TBB_full_memory_fence - // TBB has not recognized the architecture (none of the architecture abstraction - // headers was included). - #define __TBB_UnknownArchitecture 1 -#endif - -#if __TBB_UnknownArchitecture -// Implementation of atomic operations based on OS provided primitives -#include - -static inline int64_t __TBB_machine_cmpswp8_OsX(volatile void *ptr, int64_t value, int64_t comparand) -{ - __TBB_ASSERT( tbb::internal::is_aligned(ptr,8), "address not properly aligned for macOS* atomics"); - int64_t* address = (int64_t*)ptr; - while( !OSAtomicCompareAndSwap64Barrier(comparand, value, address) ){ -#if __TBB_WORDSIZE==8 - int64_t snapshot = *address; -#else - int64_t snapshot = OSAtomicAdd64( 0, address ); -#endif - if( snapshot!=comparand ) return snapshot; - } - return comparand; -} - -#define __TBB_machine_cmpswp8 __TBB_machine_cmpswp8_OsX - -#endif /* __TBB_UnknownArchitecture */ - -#if __TBB_UnknownArchitecture - -#ifndef __TBB_WORDSIZE -#define __TBB_WORDSIZE __SIZEOF_POINTER__ -#endif - -#ifdef __TBB_ENDIANNESS - // Already determined based on hardware architecture. -#elif __BIG_ENDIAN__ - #define __TBB_ENDIANNESS __TBB_ENDIAN_BIG -#elif __LITTLE_ENDIAN__ - #define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE -#else - #define __TBB_ENDIANNESS __TBB_ENDIAN_UNSUPPORTED -#endif - -/** As this generic implementation has absolutely no information about underlying - hardware, its performance most likely will be sub-optimal because of full memory - fence usages where a more lightweight synchronization means (or none at all) - could suffice. Thus if you use this header to enable TBB on a new platform, - consider forking it and relaxing below helpers as appropriate. **/ -#define __TBB_control_consistency_helper() OSMemoryBarrier() -#define __TBB_acquire_consistency_helper() OSMemoryBarrier() -#define __TBB_release_consistency_helper() OSMemoryBarrier() -#define __TBB_full_memory_fence() OSMemoryBarrier() - -static inline int32_t __TBB_machine_cmpswp4(volatile void *ptr, int32_t value, int32_t comparand) -{ - __TBB_ASSERT( tbb::internal::is_aligned(ptr,4), "address not properly aligned for macOS atomics"); - int32_t* address = (int32_t*)ptr; - while( !OSAtomicCompareAndSwap32Barrier(comparand, value, address) ){ - int32_t snapshot = *address; - if( snapshot!=comparand ) return snapshot; - } - return comparand; -} - -static inline int32_t __TBB_machine_fetchadd4(volatile void *ptr, int32_t addend) -{ - __TBB_ASSERT( tbb::internal::is_aligned(ptr,4), "address not properly aligned for macOS atomics"); - return OSAtomicAdd32Barrier(addend, (int32_t*)ptr) - addend; -} - -static inline int64_t __TBB_machine_fetchadd8(volatile void *ptr, int64_t addend) -{ - __TBB_ASSERT( tbb::internal::is_aligned(ptr,8), "address not properly aligned for macOS atomics"); - return OSAtomicAdd64Barrier(addend, (int64_t*)ptr) - addend; -} - -#define __TBB_USE_GENERIC_PART_WORD_CAS 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_ADD 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#if __TBB_WORDSIZE == 4 - #define __TBB_USE_GENERIC_DWORD_LOAD_STORE 1 -#endif -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#endif /* __TBB_UnknownArchitecture */ diff --git a/src/3rd party/tbb/machine/mic_common.h b/src/3rd party/tbb/machine/mic_common.h deleted file mode 100644 index afe0833104..0000000000 --- a/src/3rd party/tbb/machine/mic_common.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_mic_common_H -#define __TBB_mic_common_H - -#ifndef __TBB_machine_H -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#if ! __TBB_DEFINE_MIC - #error mic_common.h should be included only when building for Intel(R) Many Integrated Core Architecture -#endif - -#ifndef __TBB_PREFETCHING -#define __TBB_PREFETCHING 1 -#endif -#if __TBB_PREFETCHING -#include -#define __TBB_cl_prefetch(p) _mm_prefetch((const char*)p, _MM_HINT_T1) -#define __TBB_cl_evict(p) _mm_clevict(p, _MM_HINT_T1) -#endif - -/** Intel(R) Many Integrated Core Architecture does not support mfence and pause instructions **/ -#define __TBB_full_memory_fence() __asm__ __volatile__("lock; addl $0,(%%rsp)":::"memory") -#define __TBB_Pause(x) _mm_delay_32(16*(x)) -#define __TBB_STEALING_PAUSE 1500/16 -#include -#define __TBB_Yield() sched_yield() - -/** Specifics **/ -#define __TBB_STEALING_ABORT_ON_CONTENTION 1 -#define __TBB_YIELD2P 1 -#define __TBB_HOARD_NONLOCAL_TASKS 1 - -#if ! ( __FreeBSD__ || __linux__ ) - #error Intel(R) Many Integrated Core Compiler does not define __FreeBSD__ or __linux__ anymore. Check for the __TBB_XXX_BROKEN defined under __FreeBSD__ or __linux__. -#endif /* ! ( __FreeBSD__ || __linux__ ) */ - -#endif /* __TBB_mic_common_H */ diff --git a/src/3rd party/tbb/machine/msvc_armv7.h b/src/3rd party/tbb/machine/msvc_armv7.h deleted file mode 100644 index d38ceb9cf4..0000000000 --- a/src/3rd party/tbb/machine/msvc_armv7.h +++ /dev/null @@ -1,167 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_msvc_armv7_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_msvc_armv7_H - -#include -#include - -#define __TBB_WORDSIZE 4 - -#define __TBB_ENDIANNESS __TBB_ENDIAN_UNSUPPORTED - -#if defined(TBB_WIN32_USE_CL_BUILTINS) -// We can test this on _M_IX86 -#pragma intrinsic(_ReadWriteBarrier) -#pragma intrinsic(_mm_mfence) -#define __TBB_compiler_fence() _ReadWriteBarrier() -#define __TBB_full_memory_fence() _mm_mfence() -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence() -#else -//Now __dmb(_ARM_BARRIER_SY) is used for both compiler and memory fences -//This might be changed later after testing -#define __TBB_compiler_fence() __dmb(_ARM_BARRIER_SY) -#define __TBB_full_memory_fence() __dmb(_ARM_BARRIER_SY) -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_full_memory_fence() -#define __TBB_release_consistency_helper() __TBB_full_memory_fence() -#endif - -//-------------------------------------------------- -// Compare and swap -//-------------------------------------------------- - -/** - * Atomic CAS for 32 bit values, if *ptr==comparand, then *ptr=value, returns *ptr - * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand - * @param value value to assign *ptr to if *ptr==comparand - * @param comparand value to compare with *ptr - * @return value originally in memory at ptr, regardless of success -*/ - -#define __TBB_MACHINE_DEFINE_ATOMICS_CMPSWP(S,T,F) \ -inline T __TBB_machine_cmpswp##S( volatile void *ptr, T value, T comparand ) { \ - return _InterlockedCompareExchange##F(reinterpret_cast(ptr),value,comparand); \ -} \ - -#define __TBB_MACHINE_DEFINE_ATOMICS_FETCHADD(S,T,F) \ -inline T __TBB_machine_fetchadd##S( volatile void *ptr, T value ) { \ - return _InterlockedExchangeAdd##F(reinterpret_cast(ptr),value); \ -} \ - -__TBB_MACHINE_DEFINE_ATOMICS_CMPSWP(1,char,8) -__TBB_MACHINE_DEFINE_ATOMICS_CMPSWP(2,short,16) -__TBB_MACHINE_DEFINE_ATOMICS_CMPSWP(4,long,) -__TBB_MACHINE_DEFINE_ATOMICS_CMPSWP(8,__int64,64) -__TBB_MACHINE_DEFINE_ATOMICS_FETCHADD(4,long,) -#if defined(TBB_WIN32_USE_CL_BUILTINS) -// No _InterlockedExchangeAdd64 intrinsic on _M_IX86 -#define __TBB_64BIT_ATOMICS 0 -#else -__TBB_MACHINE_DEFINE_ATOMICS_FETCHADD(8,__int64,64) -#endif - -inline void __TBB_machine_pause (int32_t delay ) -{ - while(delay>0) - { - __TBB_compiler_fence(); - delay--; - } -} - -// API to retrieve/update FPU control setting -#define __TBB_CPU_CTL_ENV_PRESENT 1 - -namespace tbb { -namespace internal { - -template -struct machine_load_store_relaxed { - static inline T load ( const volatile T& location ) { - const T value = location; - - /* - * An extra memory barrier is required for errata #761319 - * Please see http://infocenter.arm.com/help/topic/com.arm.doc.uan0004a - */ - __TBB_acquire_consistency_helper(); - return value; - } - - static inline void store ( volatile T& location, T value ) { - location = value; - } -}; - -class cpu_ctl_env { -private: - unsigned int my_ctl; -public: - bool operator!=( const cpu_ctl_env& ctl ) const { return my_ctl != ctl.my_ctl; } - void get_env() { my_ctl = _control87(0, 0); } - void set_env() const { _control87( my_ctl, ~0U ); } -}; - -} // namespace internal -} // namespaces tbb - -// Machine specific atomic operations -#define __TBB_CompareAndSwap4(P,V,C) __TBB_machine_cmpswp4(P,V,C) -#define __TBB_CompareAndSwap8(P,V,C) __TBB_machine_cmpswp8(P,V,C) -#define __TBB_Pause(V) __TBB_machine_pause(V) - -// Use generics for some things -#define __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_ADD 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_STORE 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_DWORD_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#if defined(TBB_WIN32_USE_CL_BUILTINS) -#if !__TBB_WIN8UI_SUPPORT -extern "C" __declspec(dllimport) int __stdcall SwitchToThread( void ); -#define __TBB_Yield() SwitchToThread() -#else -#include -#define __TBB_Yield() std::this_thread::yield() -#endif -#else -#define __TBB_Yield() __yield() -#endif - -// Machine specific atomic operations -#define __TBB_AtomicOR(P,V) __TBB_machine_OR(P,V) -#define __TBB_AtomicAND(P,V) __TBB_machine_AND(P,V) - -template -inline void __TBB_machine_OR( T1 *operand, T2 addend ) { - _InterlockedOr((long volatile *)operand, (long)addend); -} - -template -inline void __TBB_machine_AND( T1 *operand, T2 addend ) { - _InterlockedAnd((long volatile *)operand, (long)addend); -} - diff --git a/src/3rd party/tbb/machine/msvc_ia32_common.h b/src/3rd party/tbb/machine/msvc_ia32_common.h deleted file mode 100644 index 541e1fa966..0000000000 --- a/src/3rd party/tbb/machine/msvc_ia32_common.h +++ /dev/null @@ -1,275 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_msvc_ia32_common_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_msvc_ia32_common_H - -#include - -//TODO: consider moving this macro to tbb_config.h and using where MSVC asm is used -#if !_M_X64 || __INTEL_COMPILER - #define __TBB_X86_MSVC_INLINE_ASM_AVAILABLE 1 -#else - //MSVC in x64 mode does not accept inline assembler - #define __TBB_X86_MSVC_INLINE_ASM_AVAILABLE 0 - #define __TBB_NO_X86_MSVC_INLINE_ASM_MSG "The compiler being used is not supported (outdated?)" -#endif - -#if _M_X64 - #define __TBB_r(reg_name) r##reg_name - #define __TBB_W(name) name##64 - namespace tbb { namespace internal { namespace msvc_intrinsics { - typedef __int64 word; - }}} -#else - #define __TBB_r(reg_name) e##reg_name - #define __TBB_W(name) name - namespace tbb { namespace internal { namespace msvc_intrinsics { - typedef long word; - }}} -#endif - -#if __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT - // S is the operand size in bytes, B is the suffix for intrinsics for that size - #define __TBB_MACHINE_DEFINE_ATOMICS(S,B,T,U) \ - __pragma(intrinsic( _InterlockedCompareExchange##B )) \ - static inline T __TBB_machine_cmpswp##S ( volatile void * ptr, U value, U comparand ) { \ - return _InterlockedCompareExchange##B ( (T*)ptr, value, comparand ); \ - } \ - __pragma(intrinsic( _InterlockedExchangeAdd##B )) \ - static inline T __TBB_machine_fetchadd##S ( volatile void * ptr, U addend ) { \ - return _InterlockedExchangeAdd##B ( (T*)ptr, addend ); \ - } \ - __pragma(intrinsic( _InterlockedExchange##B )) \ - static inline T __TBB_machine_fetchstore##S ( volatile void * ptr, U value ) { \ - return _InterlockedExchange##B ( (T*)ptr, value ); \ - } - - // Atomic intrinsics for 1, 2, and 4 bytes are available for x86 & x64 - __TBB_MACHINE_DEFINE_ATOMICS(1,8,char,__int8) - __TBB_MACHINE_DEFINE_ATOMICS(2,16,short,__int16) - __TBB_MACHINE_DEFINE_ATOMICS(4,,long,__int32) - - #if __TBB_WORDSIZE==8 - __TBB_MACHINE_DEFINE_ATOMICS(8,64,__int64,__int64) - #endif - - #undef __TBB_MACHINE_DEFINE_ATOMICS -#endif /* __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT */ - -#if _MSC_VER>=1300 || __INTEL_COMPILER>=1100 - #pragma intrinsic(_ReadWriteBarrier) - #pragma intrinsic(_mm_mfence) - #define __TBB_compiler_fence() _ReadWriteBarrier() - #define __TBB_full_memory_fence() _mm_mfence() -#elif __TBB_X86_MSVC_INLINE_ASM_AVAILABLE - #define __TBB_compiler_fence() __asm { __asm nop } - #define __TBB_full_memory_fence() __asm { __asm mfence } -#else - #error Unsupported compiler; define __TBB_{control,acquire,release}_consistency_helper to support it -#endif - -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence() - -#if (_MSC_VER>=1300) || (__INTEL_COMPILER) - #pragma intrinsic(_mm_pause) - namespace tbb { namespace internal { namespace msvc_intrinsics { - static inline void pause (uintptr_t delay ) { - for (;delay>0; --delay ) - _mm_pause(); - } - }}} - #define __TBB_Pause(V) tbb::internal::msvc_intrinsics::pause(V) - #define __TBB_SINGLE_PAUSE _mm_pause() -#else - #if !__TBB_X86_MSVC_INLINE_ASM_AVAILABLE - #error __TBB_NO_X86_MSVC_INLINE_ASM_MSG - #endif - namespace tbb { namespace internal { namespace msvc_inline_asm - static inline void pause (uintptr_t delay ) { - _asm - { - mov __TBB_r(ax), delay - __TBB_L1: - pause - add __TBB_r(ax), -1 - jne __TBB_L1 - } - return; - } - }}} - #define __TBB_Pause(V) tbb::internal::msvc_inline_asm::pause(V) - #define __TBB_SINGLE_PAUSE __asm pause -#endif - -#if (_MSC_VER>=1400 && !__INTEL_COMPILER) || (__INTEL_COMPILER>=1200) -// MSVC did not have this intrinsic prior to VC8. -// ICL 11.1 fails to compile a TBB example if __TBB_Log2 uses the intrinsic. - #pragma intrinsic(__TBB_W(_BitScanReverse)) - namespace tbb { namespace internal { namespace msvc_intrinsics { - static inline uintptr_t lg_bsr( uintptr_t i ){ - unsigned long j; - __TBB_W(_BitScanReverse)( &j, i ); - return j; - } - }}} - #define __TBB_Log2(V) tbb::internal::msvc_intrinsics::lg_bsr(V) -#else - #if !__TBB_X86_MSVC_INLINE_ASM_AVAILABLE - #error __TBB_NO_X86_MSVC_INLINE_ASM_MSG - #endif - namespace tbb { namespace internal { namespace msvc_inline_asm { - static inline uintptr_t lg_bsr( uintptr_t i ){ - uintptr_t j; - __asm - { - bsr __TBB_r(ax), i - mov j, __TBB_r(ax) - } - return j; - } - }}} - #define __TBB_Log2(V) tbb::internal::msvc_inline_asm::lg_bsr(V) -#endif - -#if _MSC_VER>=1400 - #pragma intrinsic(__TBB_W(_InterlockedOr)) - #pragma intrinsic(__TBB_W(_InterlockedAnd)) - namespace tbb { namespace internal { namespace msvc_intrinsics { - static inline void lock_or( volatile void *operand, intptr_t addend ){ - __TBB_W(_InterlockedOr)((volatile word*)operand, addend); - } - static inline void lock_and( volatile void *operand, intptr_t addend ){ - __TBB_W(_InterlockedAnd)((volatile word*)operand, addend); - } - }}} - #define __TBB_AtomicOR(P,V) tbb::internal::msvc_intrinsics::lock_or(P,V) - #define __TBB_AtomicAND(P,V) tbb::internal::msvc_intrinsics::lock_and(P,V) -#else - #if !__TBB_X86_MSVC_INLINE_ASM_AVAILABLE - #error __TBB_NO_X86_MSVC_INLINE_ASM_MSG - #endif - namespace tbb { namespace internal { namespace msvc_inline_asm { - static inline void lock_or( volatile void *operand, __int32 addend ) { - __asm - { - mov eax, addend - mov edx, [operand] - lock or [edx], eax - } - } - static inline void lock_and( volatile void *operand, __int32 addend ) { - __asm - { - mov eax, addend - mov edx, [operand] - lock and [edx], eax - } - } - }}} - #define __TBB_AtomicOR(P,V) tbb::internal::msvc_inline_asm::lock_or(P,V) - #define __TBB_AtomicAND(P,V) tbb::internal::msvc_inline_asm::lock_and(P,V) -#endif - -#pragma intrinsic(__rdtsc) -namespace tbb { namespace internal { typedef uint64_t machine_tsc_t; } } -static inline tbb::internal::machine_tsc_t __TBB_machine_time_stamp() { - return __rdtsc(); -} -#define __TBB_time_stamp() __TBB_machine_time_stamp() - -// API to retrieve/update FPU control setting -#define __TBB_CPU_CTL_ENV_PRESENT 1 - -namespace tbb { namespace internal { class cpu_ctl_env; } } -#if __TBB_X86_MSVC_INLINE_ASM_AVAILABLE - inline void __TBB_get_cpu_ctl_env ( tbb::internal::cpu_ctl_env* ctl ) { - __asm { - __asm mov __TBB_r(ax), ctl - __asm stmxcsr [__TBB_r(ax)] - __asm fstcw [__TBB_r(ax)+4] - } - } - inline void __TBB_set_cpu_ctl_env ( const tbb::internal::cpu_ctl_env* ctl ) { - __asm { - __asm mov __TBB_r(ax), ctl - __asm ldmxcsr [__TBB_r(ax)] - __asm fldcw [__TBB_r(ax)+4] - } - } -#else - extern "C" { - void __TBB_EXPORTED_FUNC __TBB_get_cpu_ctl_env ( tbb::internal::cpu_ctl_env* ); - void __TBB_EXPORTED_FUNC __TBB_set_cpu_ctl_env ( const tbb::internal::cpu_ctl_env* ); - } -#endif - -namespace tbb { -namespace internal { -class cpu_ctl_env { -private: - int mxcsr; - short x87cw; - static const int MXCSR_CONTROL_MASK = ~0x3f; /* all except last six status bits */ -public: - bool operator!=( const cpu_ctl_env& ctl ) const { return mxcsr != ctl.mxcsr || x87cw != ctl.x87cw; } - void get_env() { - __TBB_get_cpu_ctl_env( this ); - mxcsr &= MXCSR_CONTROL_MASK; - } - void set_env() const { __TBB_set_cpu_ctl_env( this ); } -}; -} // namespace internal -} // namespace tbb - -#if !__TBB_WIN8UI_SUPPORT -extern "C" __declspec(dllimport) int __stdcall SwitchToThread( void ); -#define __TBB_Yield() SwitchToThread() -#else -#include -#define __TBB_Yield() std::this_thread::yield() -#endif - -#undef __TBB_r -#undef __TBB_W -#undef __TBB_word - -extern "C" { - __int8 __TBB_EXPORTED_FUNC __TBB_machine_try_lock_elided (volatile void* ptr); - void __TBB_EXPORTED_FUNC __TBB_machine_unlock_elided (volatile void* ptr); - - // 'pause' instruction aborts HLE/RTM transactions - inline static void __TBB_machine_try_lock_elided_cancel() { __TBB_SINGLE_PAUSE; } - -#if __TBB_TSX_INTRINSICS_PRESENT - #define __TBB_machine_is_in_transaction _xtest - #define __TBB_machine_begin_transaction _xbegin - #define __TBB_machine_end_transaction _xend - // The value (0xFF) below comes from the - // Intel(R) 64 and IA-32 Architectures Optimization Reference Manual 12.4.5 lock not free - #define __TBB_machine_transaction_conflict_abort() _xabort(0xFF) -#else - __int8 __TBB_EXPORTED_FUNC __TBB_machine_is_in_transaction(); - unsigned __int32 __TBB_EXPORTED_FUNC __TBB_machine_begin_transaction(); - void __TBB_EXPORTED_FUNC __TBB_machine_end_transaction(); - void __TBB_EXPORTED_FUNC __TBB_machine_transaction_conflict_abort(); -#endif /* __TBB_TSX_INTRINSICS_PRESENT */ -} diff --git a/src/3rd party/tbb/machine/sunos_sparc.h b/src/3rd party/tbb/machine/sunos_sparc.h deleted file mode 100644 index 18e68c34fd..0000000000 --- a/src/3rd party/tbb/machine/sunos_sparc.h +++ /dev/null @@ -1,199 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_sunos_sparc_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_sunos_sparc_H - -#include -#include - -#define __TBB_WORDSIZE 8 -// Big endian is assumed for SPARC. -// While hardware may support page-specific bi-endianness, only big endian pages may be exposed to TBB -#define __TBB_ENDIANNESS __TBB_ENDIAN_BIG - -/** To those working on SPARC hardware. Consider relaxing acquire and release - consistency helpers to no-op (as this port covers TSO mode only). **/ -#define __TBB_compiler_fence() __asm__ __volatile__ ("": : :"memory") -#define __TBB_control_consistency_helper() __TBB_compiler_fence() -#define __TBB_acquire_consistency_helper() __TBB_compiler_fence() -#define __TBB_release_consistency_helper() __TBB_compiler_fence() -#define __TBB_full_memory_fence() __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreStore|#StoreLoad": : : "memory") - -//-------------------------------------------------- -// Compare and swap -//-------------------------------------------------- - -/** - * Atomic CAS for 32 bit values, if *ptr==comparand, then *ptr=value, returns *ptr - * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand - * @param value value to assign *ptr to if *ptr==comparand - * @param comparand value to compare with *ptr - ( @return value originally in memory at ptr, regardless of success -*/ -static inline int32_t __TBB_machine_cmpswp4(volatile void *ptr, int32_t value, int32_t comparand ){ - int32_t result; - __asm__ __volatile__( - "cas\t[%5],%4,%1" - : "=m"(*(int32_t *)ptr), "=r"(result) - : "m"(*(int32_t *)ptr), "1"(value), "r"(comparand), "r"(ptr) - : "memory"); - return result; -} - -/** - * Atomic CAS for 64 bit values, if *ptr==comparand, then *ptr=value, returns *ptr - * @param ptr pointer to value in memory to be swapped with value if *ptr==comparand - * @param value value to assign *ptr to if *ptr==comparand - * @param comparand value to compare with *ptr - ( @return value originally in memory at ptr, regardless of success - */ -static inline int64_t __TBB_machine_cmpswp8(volatile void *ptr, int64_t value, int64_t comparand ){ - int64_t result; - __asm__ __volatile__( - "casx\t[%5],%4,%1" - : "=m"(*(int64_t *)ptr), "=r"(result) - : "m"(*(int64_t *)ptr), "1"(value), "r"(comparand), "r"(ptr) - : "memory"); - return result; -} - -//--------------------------------------------------- -// Fetch and add -//--------------------------------------------------- - -/** - * Atomic fetch and add for 32 bit values, in this case implemented by continuously checking success of atomicity - * @param ptr pointer to value to add addend to - * @param addened value to add to *ptr - * @return value at ptr before addened was added - */ -static inline int32_t __TBB_machine_fetchadd4(volatile void *ptr, int32_t addend){ - int32_t result; - __asm__ __volatile__ ( - "0:\t add\t %3, %4, %0\n" // do addition - "\t cas\t [%2], %3, %0\n" // cas to store result in memory - "\t cmp\t %3, %0\n" // check if value from memory is original - "\t bne,a,pn\t %%icc, 0b\n" // if not try again - "\t mov %0, %3\n" // use branch delay slot to move new value in memory to be added - : "=&r"(result), "=m"(*(int32_t *)ptr) - : "r"(ptr), "r"(*(int32_t *)ptr), "r"(addend), "m"(*(int32_t *)ptr) - : "ccr", "memory"); - return result; -} - -/** - * Atomic fetch and add for 64 bit values, in this case implemented by continuously checking success of atomicity - * @param ptr pointer to value to add addend to - * @param addened value to add to *ptr - * @return value at ptr before addened was added - */ -static inline int64_t __TBB_machine_fetchadd8(volatile void *ptr, int64_t addend){ - int64_t result; - __asm__ __volatile__ ( - "0:\t add\t %3, %4, %0\n" // do addition - "\t casx\t [%2], %3, %0\n" // cas to store result in memory - "\t cmp\t %3, %0\n" // check if value from memory is original - "\t bne,a,pn\t %%xcc, 0b\n" // if not try again - "\t mov %0, %3\n" // use branch delay slot to move new value in memory to be added - : "=&r"(result), "=m"(*(int64_t *)ptr) - : "r"(ptr), "r"(*(int64_t *)ptr), "r"(addend), "m"(*(int64_t *)ptr) - : "ccr", "memory"); - return result; -} - -//-------------------------------------------------------- -// Logarithm (base two, integer) -//-------------------------------------------------------- - -static inline int64_t __TBB_machine_lg( uint64_t x ) { - __TBB_ASSERT(x, "__TBB_Log2(0) undefined"); - uint64_t count; - // one hot encode - x |= (x >> 1); - x |= (x >> 2); - x |= (x >> 4); - x |= (x >> 8); - x |= (x >> 16); - x |= (x >> 32); - // count 1's - __asm__ ("popc %1, %0" : "=r"(count) : "r"(x) ); - return count-1; -} - -//-------------------------------------------------------- - -static inline void __TBB_machine_or( volatile void *ptr, uint64_t value ) { - __asm__ __volatile__ ( - "0:\t or\t %2, %3, %%g1\n" // do operation - "\t casx\t [%1], %2, %%g1\n" // cas to store result in memory - "\t cmp\t %2, %%g1\n" // check if value from memory is original - "\t bne,a,pn\t %%xcc, 0b\n" // if not try again - "\t mov %%g1, %2\n" // use branch delay slot to move new value in memory to be added - : "=m"(*(int64_t *)ptr) - : "r"(ptr), "r"(*(int64_t *)ptr), "r"(value), "m"(*(int64_t *)ptr) - : "ccr", "g1", "memory"); -} - -static inline void __TBB_machine_and( volatile void *ptr, uint64_t value ) { - __asm__ __volatile__ ( - "0:\t and\t %2, %3, %%g1\n" // do operation - "\t casx\t [%1], %2, %%g1\n" // cas to store result in memory - "\t cmp\t %2, %%g1\n" // check if value from memory is original - "\t bne,a,pn\t %%xcc, 0b\n" // if not try again - "\t mov %%g1, %2\n" // use branch delay slot to move new value in memory to be added - : "=m"(*(int64_t *)ptr) - : "r"(ptr), "r"(*(int64_t *)ptr), "r"(value), "m"(*(int64_t *)ptr) - : "ccr", "g1", "memory"); -} - - -static inline void __TBB_machine_pause( int32_t delay ) { - // do nothing, inlined, doesn't matter -} - -// put 0xff in memory location, return memory value, -// generic trylockbyte puts 0x01, however this is fine -// because all that matters is that 0 is unlocked -static inline bool __TBB_machine_trylockbyte(unsigned char &flag){ - unsigned char result; - __asm__ __volatile__ ( - "ldstub\t [%2], %0\n" - : "=r"(result), "=m"(flag) - : "r"(&flag), "m"(flag) - : "memory"); - return result == 0; -} - -#define __TBB_USE_GENERIC_PART_WORD_CAS 1 -#define __TBB_USE_GENERIC_PART_WORD_FETCH_ADD 1 -#define __TBB_USE_GENERIC_FETCH_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - -#define __TBB_AtomicOR(P,V) __TBB_machine_or(P,V) -#define __TBB_AtomicAND(P,V) __TBB_machine_and(P,V) - -// Definition of other functions -#define __TBB_Pause(V) __TBB_machine_pause(V) -#define __TBB_Log2(V) __TBB_machine_lg(V) - -#define __TBB_TryLockByte(P) __TBB_machine_trylockbyte(P) diff --git a/src/3rd party/tbb/machine/windows_api.h b/src/3rd party/tbb/machine/windows_api.h deleted file mode 100644 index abf0e182a1..0000000000 --- a/src/3rd party/tbb/machine/windows_api.h +++ /dev/null @@ -1,65 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_machine_windows_api_H -#define __TBB_machine_windows_api_H - -#if _WIN32 || _WIN64 - -#include - -#if _WIN32_WINNT < 0x0600 -// The following Windows API function is declared explicitly; -// otherwise it fails to compile by VS2005. -#if !defined(WINBASEAPI) || (_WIN32_WINNT < 0x0501 && _MSC_VER == 1400) -#define __TBB_WINBASEAPI extern "C" -#else -#define __TBB_WINBASEAPI WINBASEAPI -#endif -__TBB_WINBASEAPI BOOL WINAPI TryEnterCriticalSection( LPCRITICAL_SECTION ); -__TBB_WINBASEAPI BOOL WINAPI InitializeCriticalSectionAndSpinCount( LPCRITICAL_SECTION, DWORD ); -// Overloading WINBASEAPI macro and using local functions missing in Windows XP/2003 -#define InitializeCriticalSectionEx inlineInitializeCriticalSectionEx -#define CreateSemaphoreEx inlineCreateSemaphoreEx -#define CreateEventEx inlineCreateEventEx -inline BOOL WINAPI inlineInitializeCriticalSectionEx( LPCRITICAL_SECTION lpCriticalSection, DWORD dwSpinCount, DWORD ) -{ - return InitializeCriticalSectionAndSpinCount( lpCriticalSection, dwSpinCount ); -} -inline HANDLE WINAPI inlineCreateSemaphoreEx( LPSECURITY_ATTRIBUTES lpSemaphoreAttributes, LONG lInitialCount, LONG lMaximumCount, LPCTSTR lpName, DWORD, DWORD ) -{ - return CreateSemaphore( lpSemaphoreAttributes, lInitialCount, lMaximumCount, lpName ); -} -inline HANDLE WINAPI inlineCreateEventEx( LPSECURITY_ATTRIBUTES lpEventAttributes, LPCTSTR lpName, DWORD dwFlags, DWORD ) -{ - BOOL manual_reset = dwFlags&0x00000001 ? TRUE : FALSE; // CREATE_EVENT_MANUAL_RESET - BOOL initial_set = dwFlags&0x00000002 ? TRUE : FALSE; // CREATE_EVENT_INITIAL_SET - return CreateEvent( lpEventAttributes, manual_reset, initial_set, lpName ); -} -#endif - -#if defined(RTL_SRWLOCK_INIT) -#ifndef __TBB_USE_SRWLOCK -// TODO: turn it on when bug 1952 will be fixed -#define __TBB_USE_SRWLOCK 0 -#endif -#endif - -#else -#error tbb/machine/windows_api.h should only be used for Windows based platforms -#endif // _WIN32 || _WIN64 - -#endif // __TBB_machine_windows_api_H diff --git a/src/3rd party/tbb/machine/windows_ia32.h b/src/3rd party/tbb/machine/windows_ia32.h deleted file mode 100644 index 86550a7e00..0000000000 --- a/src/3rd party/tbb/machine/windows_ia32.h +++ /dev/null @@ -1,105 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_windows_ia32_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_windows_ia32_H - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - // Workaround for overzealous compiler warnings in /Wp64 mode - #pragma warning (push) - #pragma warning (disable: 4244 4267) -#endif - -#include "msvc_ia32_common.h" - -#define __TBB_WORDSIZE 4 -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -extern "C" { - __int64 __TBB_EXPORTED_FUNC __TBB_machine_cmpswp8 (volatile void *ptr, __int64 value, __int64 comparand ); - __int64 __TBB_EXPORTED_FUNC __TBB_machine_fetchadd8 (volatile void *ptr, __int64 addend ); - __int64 __TBB_EXPORTED_FUNC __TBB_machine_fetchstore8 (volatile void *ptr, __int64 value ); - void __TBB_EXPORTED_FUNC __TBB_machine_store8 (volatile void *ptr, __int64 value ); - __int64 __TBB_EXPORTED_FUNC __TBB_machine_load8 (const volatile void *ptr); -} - -#if !__TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT - -#define __TBB_MACHINE_DEFINE_ATOMICS(S,T,U,A,C) \ -static inline T __TBB_machine_cmpswp##S ( volatile void * ptr, U value, U comparand ) { \ - T result; \ - volatile T *p = (T *)ptr; \ - __asm \ - { \ - __asm mov edx, p \ - __asm mov C , value \ - __asm mov A , comparand \ - __asm lock cmpxchg [edx], C \ - __asm mov result, A \ - } \ - return result; \ -} \ -\ -static inline T __TBB_machine_fetchadd##S ( volatile void * ptr, U addend ) { \ - T result; \ - volatile T *p = (T *)ptr; \ - __asm \ - { \ - __asm mov edx, p \ - __asm mov A, addend \ - __asm lock xadd [edx], A \ - __asm mov result, A \ - } \ - return result; \ -}\ -\ -static inline T __TBB_machine_fetchstore##S ( volatile void * ptr, U value ) { \ - T result; \ - volatile T *p = (T *)ptr; \ - __asm \ - { \ - __asm mov edx, p \ - __asm mov A, value \ - __asm lock xchg [edx], A \ - __asm mov result, A \ - } \ - return result; \ -} - - -__TBB_MACHINE_DEFINE_ATOMICS(1, __int8, __int8, al, cl) -__TBB_MACHINE_DEFINE_ATOMICS(2, __int16, __int16, ax, cx) -__TBB_MACHINE_DEFINE_ATOMICS(4, ptrdiff_t, ptrdiff_t, eax, ecx) - -#undef __TBB_MACHINE_DEFINE_ATOMICS - -#endif /* __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT */ - -//TODO: Check if it possible and profitable for IA-32 architecture on (Linux and Windows) -//to use of 64-bit load/store via floating point registers together with full fence -//for sequentially consistent load/store, instead of CAS. -#define __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 - - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warnings 4244, 4267 are back diff --git a/src/3rd party/tbb/machine/windows_intel64.h b/src/3rd party/tbb/machine/windows_intel64.h deleted file mode 100644 index 6b733d2018..0000000000 --- a/src/3rd party/tbb/machine/windows_intel64.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#if !defined(__TBB_machine_H) || defined(__TBB_machine_windows_intel64_H) -#error Do not #include this internal file directly; use public TBB headers instead. -#endif - -#define __TBB_machine_windows_intel64_H - -#define __TBB_WORDSIZE 8 -#define __TBB_ENDIANNESS __TBB_ENDIAN_LITTLE - -#include "msvc_ia32_common.h" - -#if !__TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT - -#include -#pragma intrinsic(_InterlockedCompareExchange,_InterlockedExchangeAdd,_InterlockedExchange) -#pragma intrinsic(_InterlockedCompareExchange64,_InterlockedExchangeAdd64,_InterlockedExchange64) - -// ATTENTION: if you ever change argument types in machine-specific primitives, -// please take care of atomic_word<> specializations in tbb/atomic.h -extern "C" { - __int8 __TBB_EXPORTED_FUNC __TBB_machine_cmpswp1 (volatile void *ptr, __int8 value, __int8 comparand ); - __int8 __TBB_EXPORTED_FUNC __TBB_machine_fetchadd1 (volatile void *ptr, __int8 addend ); - __int8 __TBB_EXPORTED_FUNC __TBB_machine_fetchstore1 (volatile void *ptr, __int8 value ); - __int16 __TBB_EXPORTED_FUNC __TBB_machine_cmpswp2 (volatile void *ptr, __int16 value, __int16 comparand ); - __int16 __TBB_EXPORTED_FUNC __TBB_machine_fetchadd2 (volatile void *ptr, __int16 addend ); - __int16 __TBB_EXPORTED_FUNC __TBB_machine_fetchstore2 (volatile void *ptr, __int16 value ); -} - -inline long __TBB_machine_cmpswp4 (volatile void *ptr, __int32 value, __int32 comparand ) { - return _InterlockedCompareExchange( (long*)ptr, value, comparand ); -} -inline long __TBB_machine_fetchadd4 (volatile void *ptr, __int32 addend ) { - return _InterlockedExchangeAdd( (long*)ptr, addend ); -} -inline long __TBB_machine_fetchstore4 (volatile void *ptr, __int32 value ) { - return _InterlockedExchange( (long*)ptr, value ); -} - -inline __int64 __TBB_machine_cmpswp8 (volatile void *ptr, __int64 value, __int64 comparand ) { - return _InterlockedCompareExchange64( (__int64*)ptr, value, comparand ); -} -inline __int64 __TBB_machine_fetchadd8 (volatile void *ptr, __int64 addend ) { - return _InterlockedExchangeAdd64( (__int64*)ptr, addend ); -} -inline __int64 __TBB_machine_fetchstore8 (volatile void *ptr, __int64 value ) { - return _InterlockedExchange64( (__int64*)ptr, value ); -} - -#endif /* __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT */ - -#define __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE 1 -#define __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_RELAXED_LOAD_STORE 1 -#define __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE 1 diff --git a/src/3rd party/tbb/memory_pool.h b/src/3rd party/tbb/memory_pool.h index 2037d634ed..cefe96e36d 100644 --- a/src/3rd party/tbb/memory_pool.h +++ b/src/3rd party/tbb/memory_pool.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,262 +14,4 @@ limitations under the License. */ -#ifndef __TBB_memory_pool_H -#define __TBB_memory_pool_H - -#if !TBB_PREVIEW_MEMORY_POOL -#error Set TBB_PREVIEW_MEMORY_POOL to include memory_pool.h -#endif -/** @file */ - -#include "scalable_allocator.h" -#include // std::bad_alloc -#include // std::runtime_error, std::invalid_argument -// required in C++03 to construct std::runtime_error and std::invalid_argument -#include -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC -#include // std::forward -#endif - -#if __TBB_EXTRA_DEBUG -#define __TBBMALLOC_ASSERT ASSERT -#else -#define __TBBMALLOC_ASSERT(a,b) ((void)0) -#endif - -namespace tbb { -namespace interface6 { -//! @cond INTERNAL -namespace internal { - -//! Base of thread-safe pool allocator for variable-size requests -class pool_base : tbb::internal::no_copy { - // Pool interface is separate from standard allocator classes because it has - // to maintain internal state, no copy or assignment. Move and swap are possible. -public: - //! Reset pool to reuse its memory (free all objects at once) - void recycle() { rml::pool_reset(my_pool); } - - //! The "malloc" analogue to allocate block of memory of size bytes - void *malloc(size_t size) { return rml::pool_malloc(my_pool, size); } - - //! The "free" analogue to discard a previously allocated piece of memory. - void free(void* ptr) { rml::pool_free(my_pool, ptr); } - - //! The "realloc" analogue complementing pool_malloc. - // Enables some low-level optimization possibilities - void *realloc(void* ptr, size_t size) { - return rml::pool_realloc(my_pool, ptr, size); - } - -protected: - //! destroy pool - must be called in a child class - void destroy() { rml::pool_destroy(my_pool); } - - rml::MemoryPool *my_pool; -}; - -} // namespace internal -//! @endcond - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Workaround for erroneous "unreferenced parameter" warning in method destroy. - #pragma warning (push) - #pragma warning (disable: 4100) -#endif - -//! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 -/** @ingroup memory_allocation */ -template -class memory_pool_allocator { -protected: - typedef P pool_type; - pool_type *my_pool; - template - friend class memory_pool_allocator; - template - friend bool operator==( const memory_pool_allocator& a, const memory_pool_allocator& b); - template - friend bool operator!=( const memory_pool_allocator& a, const memory_pool_allocator& b); -public: - typedef typename tbb::internal::allocator_type::value_type value_type; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - template struct rebind { - typedef memory_pool_allocator other; - }; - - explicit memory_pool_allocator(pool_type &pool) throw() : my_pool(&pool) {} - memory_pool_allocator(const memory_pool_allocator& src) throw() : my_pool(src.my_pool) {} - template - memory_pool_allocator(const memory_pool_allocator& src) throw() : my_pool(src.my_pool) {} - - pointer address(reference x) const { return &x; } - const_pointer address(const_reference x) const { return &x; } - - //! Allocate space for n objects. - pointer allocate( size_type n, const void* /*hint*/ = 0) { - pointer p = static_cast( my_pool->malloc( n*sizeof(value_type) ) ); - if (!p) - tbb::internal::throw_exception(std::bad_alloc()); - return p; - } - //! Free previously allocated block of memory. - void deallocate( pointer p, size_type ) { - my_pool->free(p); - } - //! Largest value for which method allocate might succeed. - size_type max_size() const throw() { - size_type max = static_cast(-1) / sizeof (value_type); - return (max > 0 ? max : 1); - } - //! Copy-construct value at location pointed to by p. -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - template - void construct(U *p, Args&&... args) - { ::new((void *)p) U(std::forward(args)...); } -#else // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC -#if __TBB_CPP11_RVALUE_REF_PRESENT - void construct( pointer p, value_type&& value ) {::new((void*)(p)) value_type(std::move(value));} -#endif - void construct( pointer p, const value_type& value ) { ::new((void*)(p)) value_type(value); } -#endif // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - - //! Destroy value at location pointed to by p. - void destroy( pointer p ) { p->~value_type(); } - -}; - -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warning 4100 is back - -//! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1 -/** @ingroup memory_allocation */ -template -class memory_pool_allocator { -public: - typedef P pool_type; - typedef void* pointer; - typedef const void* const_pointer; - typedef void value_type; - template struct rebind { - typedef memory_pool_allocator other; - }; - - explicit memory_pool_allocator( pool_type &pool) throw() : my_pool(&pool) {} - memory_pool_allocator( const memory_pool_allocator& src) throw() : my_pool(src.my_pool) {} - template - memory_pool_allocator(const memory_pool_allocator& src) throw() : my_pool(src.my_pool) {} - -protected: - pool_type *my_pool; - template - friend class memory_pool_allocator; - template - friend bool operator==( const memory_pool_allocator& a, const memory_pool_allocator& b); - template - friend bool operator!=( const memory_pool_allocator& a, const memory_pool_allocator& b); -}; - -template -inline bool operator==( const memory_pool_allocator& a, const memory_pool_allocator& b) {return a.my_pool==b.my_pool;} - -template -inline bool operator!=( const memory_pool_allocator& a, const memory_pool_allocator& b) {return a.my_pool!=b.my_pool;} - - -//! Thread-safe growable pool allocator for variable-size requests -template -class memory_pool : public internal::pool_base { - Alloc my_alloc; // TODO: base-class optimization - static void *allocate_request(intptr_t pool_id, size_t & bytes); - static int deallocate_request(intptr_t pool_id, void*, size_t raw_bytes); - -public: - //! construct pool with underlying allocator - explicit memory_pool(const Alloc &src = Alloc()); - - //! destroy pool - ~memory_pool() { destroy(); } // call the callbacks first and destroy my_alloc latter - -}; - -class fixed_pool : public internal::pool_base { - void *my_buffer; - size_t my_size; - inline static void *allocate_request(intptr_t pool_id, size_t & bytes); - -public: - //! construct pool with underlying allocator - inline fixed_pool(void *buf, size_t size); - //! destroy pool - ~fixed_pool() { destroy(); } -}; - -//////////////// Implementation /////////////// - -template -memory_pool::memory_pool(const Alloc &src) : my_alloc(src) { - rml::MemPoolPolicy args(allocate_request, deallocate_request, - sizeof(typename Alloc::value_type)); - rml::MemPoolError res = rml::pool_create_v1(intptr_t(this), &args, &my_pool); - if (res!=rml::POOL_OK) - tbb::internal::throw_exception(std::runtime_error("Can't create pool")); -} -template -void *memory_pool::allocate_request(intptr_t pool_id, size_t & bytes) { - memory_pool &self = *reinterpret_cast*>(pool_id); - const size_t unit_size = sizeof(typename Alloc::value_type); - __TBBMALLOC_ASSERT( 0 == bytes%unit_size, NULL); - void *ptr; - __TBB_TRY { ptr = self.my_alloc.allocate( bytes/unit_size ); } - __TBB_CATCH(...) { return 0; } - return ptr; -} -#if __TBB_MSVC_UNREACHABLE_CODE_IGNORED - // Workaround for erroneous "unreachable code" warning in the template below. - // Specific for VC++ 17-18 compiler - #pragma warning (push) - #pragma warning (disable: 4702) -#endif -template -int memory_pool::deallocate_request(intptr_t pool_id, void* raw_ptr, size_t raw_bytes) { - memory_pool &self = *reinterpret_cast*>(pool_id); - const size_t unit_size = sizeof(typename Alloc::value_type); - __TBBMALLOC_ASSERT( 0 == raw_bytes%unit_size, NULL); - self.my_alloc.deallocate( static_cast(raw_ptr), raw_bytes/unit_size ); - return 0; -} -#if __TBB_MSVC_UNREACHABLE_CODE_IGNORED - #pragma warning (pop) -#endif -inline fixed_pool::fixed_pool(void *buf, size_t size) : my_buffer(buf), my_size(size) { - if (!buf || !size) - // TODO: improve support for mode with exceptions disabled - tbb::internal::throw_exception(std::invalid_argument("Zero in parameter is invalid")); - rml::MemPoolPolicy args(allocate_request, 0, size, /*fixedPool=*/true); - rml::MemPoolError res = rml::pool_create_v1(intptr_t(this), &args, &my_pool); - if (res!=rml::POOL_OK) - tbb::internal::throw_exception(std::runtime_error("Can't create pool")); -} -inline void *fixed_pool::allocate_request(intptr_t pool_id, size_t & bytes) { - fixed_pool &self = *reinterpret_cast(pool_id); - __TBBMALLOC_ASSERT(0 != self.my_size, "The buffer must not be used twice."); - bytes = self.my_size; - self.my_size = 0; // remember that buffer has been used - return self.my_buffer; -} - -} //namespace interface6 -using interface6::memory_pool_allocator; -using interface6::memory_pool; -using interface6::fixed_pool; -} //namespace tbb - -#undef __TBBMALLOC_ASSERT -#endif// __TBB_memory_pool_H +#include "../oneapi/tbb/memory_pool.h" diff --git a/src/3rd party/tbb/mutex.h b/src/3rd party/tbb/mutex.h index ca46ec806a..91dbee0fa0 100644 --- a/src/3rd party/tbb/mutex.h +++ b/src/3rd party/tbb/mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2023 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,233 +14,4 @@ limitations under the License. */ -#include "internal/_deprecated_header_message_guard.h" - -#if !defined(__TBB_show_deprecation_message_mutex_H) && defined(__TBB_show_deprecated_header_message) -#define __TBB_show_deprecation_message_mutex_H -#pragma message("TBB Warning: tbb/mutex.h is deprecated. For details, please see Deprecated Features appendix in the TBB reference manual.") -#endif - -#if defined(__TBB_show_deprecated_header_message) -#undef __TBB_show_deprecated_header_message -#endif - -#ifndef __TBB_mutex_H -#define __TBB_mutex_H - -#define __TBB_mutex_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#else -#include -#endif /* _WIN32||_WIN64 */ - -#include -#include "aligned_space.h" -#include "tbb_stddef.h" -#include "tbb_profiling.h" - -namespace tbb { - -//! Wrapper around the platform's native lock. -/** @ingroup synchronization */ -class __TBB_DEPRECATED_VERBOSE_MSG("tbb::critical_section is deprecated, use std::mutex") mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Construct unacquired mutex. - mutex() { -#if TBB_USE_ASSERT || TBB_USE_THREADING_TOOLS - internal_construct(); -#else - #if _WIN32||_WIN64 - InitializeCriticalSectionEx(&impl, 4000, 0); - #else - int error_code = pthread_mutex_init(&impl,NULL); - if( error_code ) - tbb::internal::handle_perror(error_code,"mutex: pthread_mutex_init failed"); - #endif /* _WIN32||_WIN64*/ -#endif /* TBB_USE_ASSERT */ - }; - - ~mutex() { -#if TBB_USE_ASSERT - internal_destroy(); -#else - #if _WIN32||_WIN64 - DeleteCriticalSection(&impl); - #else - pthread_mutex_destroy(&impl); - - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - }; - - class scoped_lock; - friend class scoped_lock; - - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - class scoped_lock : internal::no_copy { - public: - //! Construct lock that has not acquired a mutex. - scoped_lock() : my_mutex(NULL) {}; - - //! Acquire lock on given mutex. - scoped_lock( mutex& mutex ) { - acquire( mutex ); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if( my_mutex ) - release(); - } - - //! Acquire lock on given mutex. - void acquire( mutex& mutex ) { -#if TBB_USE_ASSERT - internal_acquire(mutex); -#else - mutex.lock(); - my_mutex = &mutex; -#endif /* TBB_USE_ASSERT */ - } - - //! Try acquire lock on given mutex. - bool try_acquire( mutex& mutex ) { -#if TBB_USE_ASSERT - return internal_try_acquire (mutex); -#else - bool result = mutex.try_lock(); - if( result ) - my_mutex = &mutex; - return result; -#endif /* TBB_USE_ASSERT */ - } - - //! Release lock - void release() { -#if TBB_USE_ASSERT - internal_release (); -#else - my_mutex->unlock(); - my_mutex = NULL; -#endif /* TBB_USE_ASSERT */ - } - - private: - //! The pointer to the current mutex to work - mutex* my_mutex; - - //! All checks from acquire using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_acquire( mutex& m ); - - //! All checks from try_acquire using mutex.state were moved here - bool __TBB_EXPORTED_METHOD internal_try_acquire( mutex& m ); - - //! All checks from release using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_release(); - - friend class mutex; - }; - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = false; - - // ISO C++0x compatibility methods - - //! Acquire lock - void lock() { -#if TBB_USE_ASSERT - aligned_space tmp; - new(tmp.begin()) scoped_lock(*this); -#else - #if _WIN32||_WIN64 - EnterCriticalSection(&impl); - #else - int error_code = pthread_mutex_lock(&impl); - if( error_code ) - tbb::internal::handle_perror(error_code,"mutex: pthread_mutex_lock failed"); - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_lock() { -#if TBB_USE_ASSERT - aligned_space tmp; - scoped_lock& s = *tmp.begin(); - s.my_mutex = NULL; - return s.internal_try_acquire(*this); -#else - #if _WIN32||_WIN64 - return TryEnterCriticalSection(&impl)!=0; - #else - return pthread_mutex_trylock(&impl)==0; - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Release lock - void unlock() { -#if TBB_USE_ASSERT - aligned_space tmp; - scoped_lock& s = *tmp.begin(); - s.my_mutex = this; - s.internal_release(); -#else - #if _WIN32||_WIN64 - LeaveCriticalSection(&impl); - #else - pthread_mutex_unlock(&impl); - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Return native_handle - #if _WIN32||_WIN64 - typedef LPCRITICAL_SECTION native_handle_type; - #else - typedef pthread_mutex_t* native_handle_type; - #endif - native_handle_type native_handle() { return (native_handle_type) &impl; } - - enum state_t { - INITIALIZED=0x1234, - DESTROYED=0x789A, - HELD=0x56CD - }; -private: -#if _WIN32||_WIN64 - CRITICAL_SECTION impl; - enum state_t state; -#else - pthread_mutex_t impl; -#endif /* _WIN32||_WIN64 */ - - //! All checks from mutex constructor using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_construct(); - - //! All checks from mutex destructor using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_destroy(); - -#if _WIN32||_WIN64 -public: - //! Set the internal state - void set_state( state_t to ) { state = to; } -#endif -}; - -__TBB_DEFINE_PROFILING_SET_NAME(mutex) - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_mutex_H_include_area - -#endif /* __TBB_mutex_H */ +#include "../oneapi/tbb/mutex.h" diff --git a/src/3rd party/tbb/null_mutex.h b/src/3rd party/tbb/null_mutex.h index b9c382dd23..63218bf061 100644 --- a/src/3rd party/tbb/null_mutex.h +++ b/src/3rd party/tbb/null_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,37 +14,4 @@ limitations under the License. */ -#ifndef __TBB_null_mutex_H -#define __TBB_null_mutex_H - -#include "tbb_stddef.h" - -namespace tbb { - -//! A mutex which does nothing -/** A null_mutex does no operation and simulates success. - @ingroup synchronization */ -class null_mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Represents acquisition of a mutex. - class scoped_lock : internal::no_copy { - public: - scoped_lock() {} - scoped_lock( null_mutex& ) {} - ~scoped_lock() {} - void acquire( null_mutex& ) {} - bool try_acquire( null_mutex& ) { return true; } - void release() {} - }; - - null_mutex() {} - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = true; - static const bool is_fair_mutex = true; -}; - -} - -#endif /* __TBB_null_mutex_H */ +#include "../oneapi/tbb/null_mutex.h" diff --git a/src/3rd party/tbb/null_rw_mutex.h b/src/3rd party/tbb/null_rw_mutex.h index d2fd33ca2a..71c42fe26a 100644 --- a/src/3rd party/tbb/null_rw_mutex.h +++ b/src/3rd party/tbb/null_rw_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,39 +14,4 @@ limitations under the License. */ -#ifndef __TBB_null_rw_mutex_H -#define __TBB_null_rw_mutex_H - -#include "tbb_stddef.h" - -namespace tbb { - -//! A rw mutex which does nothing -/** A null_rw_mutex is a rw mutex that does nothing and simulates successful operation. - @ingroup synchronization */ -class null_rw_mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Represents acquisition of a mutex. - class scoped_lock : internal::no_copy { - public: - scoped_lock() {} - scoped_lock( null_rw_mutex& , bool = true ) {} - ~scoped_lock() {} - void acquire( null_rw_mutex& , bool = true ) {} - bool upgrade_to_writer() { return true; } - bool downgrade_to_reader() { return true; } - bool try_acquire( null_rw_mutex& , bool = true ) { return true; } - void release() {} - }; - - null_rw_mutex() {} - - // Mutex traits - static const bool is_rw_mutex = true; - static const bool is_recursive_mutex = true; - static const bool is_fair_mutex = true; -}; - -} - -#endif /* __TBB_null_rw_mutex_H */ +#include "../oneapi/tbb/null_rw_mutex.h" diff --git a/src/3rd party/tbb/parallel_do.h b/src/3rd party/tbb/parallel_do.h deleted file mode 100644 index ab0f151b5c..0000000000 --- a/src/3rd party/tbb/parallel_do.h +++ /dev/null @@ -1,553 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_parallel_do_H -#define __TBB_parallel_do_H - -#define __TBB_parallel_do_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "internal/_range_iterator.h" -#include "internal/_template_helpers.h" -#include "task.h" -#include "aligned_space.h" -#include - -namespace tbb { -namespace interface9 { -//! @cond INTERNAL -namespace internal { - template class parallel_do_feeder_impl; -} // namespace internal -//! @endcond - -//! Class the user supplied algorithm body uses to add new tasks -/** \param Item Work item type **/ - template - class parallel_do_feeder: ::tbb::internal::no_copy - { - parallel_do_feeder() {} - virtual ~parallel_do_feeder () {} - virtual void internal_add_copy( const Item& item ) = 0; -#if __TBB_CPP11_RVALUE_REF_PRESENT - virtual void internal_add_move( Item&& item ) = 0; -#endif - template friend class internal::parallel_do_feeder_impl; - public: - //! Add a work item to a running parallel_do. - void add( const Item& item ) {internal_add_copy(item);} -#if __TBB_CPP11_RVALUE_REF_PRESENT - void add( Item&& item ) {internal_add_move(std::move(item));} -#endif - }; - -//! @cond INTERNAL -namespace internal { - template class do_group_task; - - //! For internal use only. - /** Selects one of the two possible forms of function call member operator. - @ingroup algorithms **/ - template - class parallel_do_operator_selector - { - typedef parallel_do_feeder Feeder; - template - static void internal_call( const Body& obj, __TBB_FORWARDING_REF(A1) arg1, A2&, void (Body::*)(CvItem) const ) { - obj(tbb::internal::forward(arg1)); - } - template - static void internal_call( const Body& obj, __TBB_FORWARDING_REF(A1) arg1, A2& arg2, void (Body::*)(CvItem, parallel_do_feeder&) const ) { - obj(tbb::internal::forward(arg1), arg2); - } - template - static void internal_call( const Body& obj, __TBB_FORWARDING_REF(A1) arg1, A2&, void (Body::*)(CvItem&) const ) { - obj(arg1); - } - template - static void internal_call( const Body& obj, __TBB_FORWARDING_REF(A1) arg1, A2& arg2, void (Body::*)(CvItem&, parallel_do_feeder&) const ) { - obj(arg1, arg2); - } - public: - template - static void call( const Body& obj, __TBB_FORWARDING_REF(A1) arg1, A2& arg2 ) - { - internal_call( obj, tbb::internal::forward(arg1), arg2, &Body::operator() ); - } - }; - - //! For internal use only. - /** Executes one iteration of a do. - @ingroup algorithms */ - template - class do_iteration_task: public task - { - typedef parallel_do_feeder_impl feeder_type; - - Item my_value; - feeder_type& my_feeder; - - do_iteration_task( const Item& value, feeder_type& feeder ) : - my_value(value), my_feeder(feeder) - {} - -#if __TBB_CPP11_RVALUE_REF_PRESENT - do_iteration_task( Item&& value, feeder_type& feeder ) : - my_value(std::move(value)), my_feeder(feeder) - {} -#endif - - task* execute() __TBB_override - { - parallel_do_operator_selector::call(*my_feeder.my_body, tbb::internal::move(my_value), my_feeder); - return NULL; - } - - template friend class parallel_do_feeder_impl; - }; // class do_iteration_task - - template - class do_iteration_task_iter: public task - { - typedef parallel_do_feeder_impl feeder_type; - - Iterator my_iter; - feeder_type& my_feeder; - - do_iteration_task_iter( const Iterator& iter, feeder_type& feeder ) : - my_iter(iter), my_feeder(feeder) - {} - - task* execute() __TBB_override - { - parallel_do_operator_selector::call(*my_feeder.my_body, *my_iter, my_feeder); - return NULL; - } - - template friend class do_group_task_forward; - template friend class do_group_task_input; - template friend class do_task_iter; - }; // class do_iteration_task_iter - - //! For internal use only. - /** Implements new task adding procedure. - @ingroup algorithms **/ - template - class parallel_do_feeder_impl : public parallel_do_feeder - { -#if __TBB_CPP11_RVALUE_REF_PRESENT - //Avoiding use of copy constructor in a virtual method if the type does not support it - void internal_add_copy_impl(std::true_type, const Item& item) { - typedef do_iteration_task iteration_type; - iteration_type& t = *new (task::allocate_additional_child_of(*my_barrier)) iteration_type(item, *this); - task::spawn(t); - } - void internal_add_copy_impl(std::false_type, const Item&) { - __TBB_ASSERT(false, "Overloading for r-value reference doesn't work or it's not movable and not copyable object"); - } - void internal_add_copy( const Item& item ) __TBB_override - { -#if __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT - internal_add_copy_impl(typename std::is_copy_constructible::type(), item); -#else - internal_add_copy_impl(std::true_type(), item); -#endif - } - void internal_add_move( Item&& item ) __TBB_override - { - typedef do_iteration_task iteration_type; - iteration_type& t = *new (task::allocate_additional_child_of(*my_barrier)) iteration_type(std::move(item), *this); - task::spawn(t); - } -#else /* ! __TBB_CPP11_RVALUE_REF_PRESENT */ - void internal_add_copy(const Item& item) __TBB_override { - typedef do_iteration_task iteration_type; - iteration_type& t = *new (task::allocate_additional_child_of(*my_barrier)) iteration_type(item, *this); - task::spawn(t); - } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - public: - const Body* my_body; - empty_task* my_barrier; - - parallel_do_feeder_impl() - { - my_barrier = new( task::allocate_root() ) empty_task(); - __TBB_ASSERT(my_barrier, "root task allocation failed"); - } - -#if __TBB_TASK_GROUP_CONTEXT - parallel_do_feeder_impl(tbb::task_group_context &context) - { - my_barrier = new( task::allocate_root(context) ) empty_task(); - __TBB_ASSERT(my_barrier, "root task allocation failed"); - } -#endif - - ~parallel_do_feeder_impl() - { - my_barrier->destroy(*my_barrier); - } - }; // class parallel_do_feeder_impl - - - //! For internal use only - /** Unpacks a block of iterations. - @ingroup algorithms */ - - template - class do_group_task_forward: public task - { - static const size_t max_arg_size = 4; - - typedef parallel_do_feeder_impl feeder_type; - - feeder_type& my_feeder; - Iterator my_first; - size_t my_size; - - do_group_task_forward( Iterator first, size_t size, feeder_type& feeder ) - : my_feeder(feeder), my_first(first), my_size(size) - {} - - task* execute() __TBB_override - { - typedef do_iteration_task_iter iteration_type; - __TBB_ASSERT( my_size>0, NULL ); - task_list list; - task* t; - size_t k=0; - for(;;) { - t = new( allocate_child() ) iteration_type( my_first, my_feeder ); - ++my_first; - if( ++k==my_size ) break; - list.push_back(*t); - } - set_ref_count(int(k+1)); - spawn(list); - spawn_and_wait_for_all(*t); - return NULL; - } - - template friend class do_task_iter; - }; // class do_group_task_forward - - template - class do_group_task_input: public task - { - static const size_t max_arg_size = 4; - - typedef parallel_do_feeder_impl feeder_type; - - feeder_type& my_feeder; - size_t my_size; - aligned_space my_arg; - - do_group_task_input( feeder_type& feeder ) - : my_feeder(feeder), my_size(0) - {} - - task* execute() __TBB_override - { -#if __TBB_CPP11_RVALUE_REF_PRESENT - typedef std::move_iterator Item_iterator; -#else - typedef Item* Item_iterator; -#endif - typedef do_iteration_task_iter iteration_type; - __TBB_ASSERT( my_size>0, NULL ); - task_list list; - task* t; - size_t k=0; - for(;;) { - t = new( allocate_child() ) iteration_type( Item_iterator(my_arg.begin() + k), my_feeder ); - if( ++k==my_size ) break; - list.push_back(*t); - } - set_ref_count(int(k+1)); - spawn(list); - spawn_and_wait_for_all(*t); - return NULL; - } - - ~do_group_task_input(){ - for( size_t k=0; k~Item(); - } - - template friend class do_task_iter; - }; // class do_group_task_input - - //! For internal use only. - /** Gets block of iterations and packages them into a do_group_task. - @ingroup algorithms */ - template - class do_task_iter: public task - { - typedef parallel_do_feeder_impl feeder_type; - - public: - do_task_iter( Iterator first, Iterator last , feeder_type& feeder ) : - my_first(first), my_last(last), my_feeder(feeder) - {} - - private: - Iterator my_first; - Iterator my_last; - feeder_type& my_feeder; - - /* Do not merge run(xxx) and run_xxx() methods. They are separated in order - to make sure that compilers will eliminate unused argument of type xxx - (that is will not put it on stack). The sole purpose of this argument - is overload resolution. - - An alternative could be using template functions, but explicit specialization - of member function templates is not supported for non specialized class - templates. Besides template functions would always fall back to the least - efficient variant (the one for input iterators) in case of iterators having - custom tags derived from basic ones. */ - task* execute() __TBB_override - { - typedef typename std::iterator_traits::iterator_category iterator_tag; - return run( (iterator_tag*)NULL ); - } - - /** This is the most restricted variant that operates on input iterators or - iterators with unknown tags (tags not derived from the standard ones). **/ - inline task* run( void* ) { return run_for_input_iterator(); } - - task* run_for_input_iterator() { - typedef do_group_task_input block_type; - - block_type& t = *new( allocate_additional_child_of(*my_feeder.my_barrier) ) block_type(my_feeder); - size_t k=0; - while( !(my_first == my_last) ) { - // Move semantics are automatically used when supported by the iterator - new (t.my_arg.begin() + k) Item(*my_first); - ++my_first; - if( ++k==block_type::max_arg_size ) { - if ( !(my_first == my_last) ) - recycle_to_reexecute(); - break; - } - } - if( k==0 ) { - destroy(t); - return NULL; - } else { - t.my_size = k; - return &t; - } - } - - inline task* run( std::forward_iterator_tag* ) { return run_for_forward_iterator(); } - - task* run_for_forward_iterator() { - typedef do_group_task_forward block_type; - - Iterator first = my_first; - size_t k=0; - while( !(my_first==my_last) ) { - ++my_first; - if( ++k==block_type::max_arg_size ) { - if ( !(my_first==my_last) ) - recycle_to_reexecute(); - break; - } - } - return k==0 ? NULL : new( allocate_additional_child_of(*my_feeder.my_barrier) ) block_type(first, k, my_feeder); - } - - inline task* run( std::random_access_iterator_tag* ) { return run_for_random_access_iterator(); } - - task* run_for_random_access_iterator() { - typedef do_group_task_forward block_type; - typedef do_iteration_task_iter iteration_type; - - size_t k = static_cast(my_last-my_first); - if( k > block_type::max_arg_size ) { - Iterator middle = my_first + k/2; - - empty_task& c = *new( allocate_continuation() ) empty_task; - do_task_iter& b = *new( c.allocate_child() ) do_task_iter(middle, my_last, my_feeder); - recycle_as_child_of(c); - - my_last = middle; - c.set_ref_count(2); - c.spawn(b); - return this; - }else if( k != 0 ) { - task_list list; - task* t; - size_t k1=0; - for(;;) { - t = new( allocate_child() ) iteration_type(my_first, my_feeder); - ++my_first; - if( ++k1==k ) break; - list.push_back(*t); - } - set_ref_count(int(k+1)); - spawn(list); - spawn_and_wait_for_all(*t); - } - return NULL; - } - }; // class do_task_iter - - //! For internal use only. - /** Implements parallel iteration over a range. - @ingroup algorithms */ - template - void run_parallel_do( Iterator first, Iterator last, const Body& body -#if __TBB_TASK_GROUP_CONTEXT - , task_group_context& context -#endif - ) - { - typedef do_task_iter root_iteration_task; -#if __TBB_TASK_GROUP_CONTEXT - parallel_do_feeder_impl feeder(context); -#else - parallel_do_feeder_impl feeder; -#endif - feeder.my_body = &body; - - root_iteration_task &t = *new( feeder.my_barrier->allocate_child() ) root_iteration_task(first, last, feeder); - - feeder.my_barrier->set_ref_count(2); - feeder.my_barrier->spawn_and_wait_for_all(t); - } - - //! For internal use only. - /** Detects types of Body's operator function arguments. - @ingroup algorithms **/ - template - void select_parallel_do( Iterator first, Iterator last, const Body& body, void (Body::*)(Item) const -#if __TBB_TASK_GROUP_CONTEXT - , task_group_context& context -#endif - ) - { - run_parallel_do::type>( first, last, body -#if __TBB_TASK_GROUP_CONTEXT - , context -#endif - ); - } - - //! For internal use only. - /** Detects types of Body's operator function arguments. - @ingroup algorithms **/ - template - void select_parallel_do( Iterator first, Iterator last, const Body& body, void (Body::*)(Item, parallel_do_feeder<_Item>&) const -#if __TBB_TASK_GROUP_CONTEXT - , task_group_context& context -#endif - ) - { - run_parallel_do::type>( first, last, body -#if __TBB_TASK_GROUP_CONTEXT - , context -#endif - ); - } - -} // namespace internal -} // namespace interface9 -//! @endcond - -/** \page parallel_do_body_req Requirements on parallel_do body - Class \c Body implementing the concept of parallel_do body must define: - - \code - B::operator()( - cv_item_type item, - parallel_do_feeder& feeder - ) const - - OR - - B::operator()( cv_item_type& item ) const - \endcode Process item. - May be invoked concurrently for the same \c this but different \c item. - - - \code item_type( const item_type& ) \endcode - Copy a work item. - - \code ~item_type() \endcode Destroy a work item -**/ - -/** \name parallel_do - See also requirements on \ref parallel_do_body_req "parallel_do Body". **/ -//@{ -//! Parallel iteration over a range, with optional addition of more work. -/** @ingroup algorithms */ -template -void parallel_do( Iterator first, Iterator last, const Body& body ) -{ - if ( first == last ) - return; -#if __TBB_TASK_GROUP_CONTEXT - task_group_context context(internal::PARALLEL_DO); -#endif - interface9::internal::select_parallel_do( first, last, body, &Body::operator() -#if __TBB_TASK_GROUP_CONTEXT - , context -#endif - ); -} - -template -void parallel_do(Range& rng, const Body& body) { - parallel_do(tbb::internal::first(rng), tbb::internal::last(rng), body); -} - -template -void parallel_do(const Range& rng, const Body& body) { - parallel_do(tbb::internal::first(rng), tbb::internal::last(rng), body); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration over a range, with optional addition of more work and user-supplied context -/** @ingroup algorithms */ -template -void parallel_do( Iterator first, Iterator last, const Body& body, task_group_context& context ) -{ - if ( first == last ) - return; - interface9::internal::select_parallel_do( first, last, body, &Body::operator(), context ); -} - -template -void parallel_do(Range& rng, const Body& body, task_group_context& context) { - parallel_do(tbb::internal::first(rng), tbb::internal::last(rng), body, context); -} - -template -void parallel_do(const Range& rng, const Body& body, task_group_context& context) { - parallel_do(tbb::internal::first(rng), tbb::internal::last(rng), body, context); -} - -#endif // __TBB_TASK_GROUP_CONTEXT - -//@} - -using interface9::parallel_do_feeder; - -} // namespace - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_parallel_do_H_include_area - -#endif /* __TBB_parallel_do_H */ diff --git a/src/3rd party/tbb/parallel_for.h b/src/3rd party/tbb/parallel_for.h index bcdcaad33e..fea1d1b9f5 100644 --- a/src/3rd party/tbb/parallel_for.h +++ b/src/3rd party/tbb/parallel_for.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,412 +14,4 @@ limitations under the License. */ -#ifndef __TBB_parallel_for_H -#define __TBB_parallel_for_H - -#define __TBB_parallel_for_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include -#include "task.h" -#include "partitioner.h" -#include "blocked_range.h" -#include "tbb_exception.h" -#include "internal/_tbb_trace_impl.h" - -namespace tbb { - -namespace interface9 { -//! @cond INTERNAL -namespace internal { - - //! allocate right task with new parent - void* allocate_sibling(task* start_for_task, size_t bytes); - - //! Task type used in parallel_for - /** @ingroup algorithms */ - template - class start_for: public task { - Range my_range; - const Body my_body; - typename Partitioner::task_partition_type my_partition; - task* execute() __TBB_override; - - //! Update affinity info, if any. - void note_affinity( affinity_id id ) __TBB_override { - my_partition.note_affinity( id ); - } - - public: - //! Constructor for root task. - start_for( const Range& range, const Body& body, Partitioner& partitioner ) : - my_range(range), - my_body(body), - my_partition(partitioner) - { - tbb::internal::fgt_algorithm(tbb::internal::PARALLEL_FOR_TASK, this, NULL); - } - //! Splitting constructor used to generate children. - /** parent_ becomes left child. Newly constructed object is right child. */ - start_for( start_for& parent_, typename Partitioner::split_type& split_obj) : - my_range(parent_.my_range, split_obj), - my_body(parent_.my_body), - my_partition(parent_.my_partition, split_obj) - { - my_partition.set_affinity(*this); - tbb::internal::fgt_algorithm(tbb::internal::PARALLEL_FOR_TASK, this, (void *)&parent_); - } - //! Construct right child from the given range as response to the demand. - /** parent_ remains left child. Newly constructed object is right child. */ - start_for( start_for& parent_, const Range& r, depth_t d ) : - my_range(r), - my_body(parent_.my_body), - my_partition(parent_.my_partition, split()) - { - my_partition.set_affinity(*this); - my_partition.align_depth( d ); - tbb::internal::fgt_algorithm(tbb::internal::PARALLEL_FOR_TASK, this, (void *)&parent_); - } - static void run( const Range& range, const Body& body, Partitioner& partitioner ) { - if( !range.empty() ) { -#if !__TBB_TASK_GROUP_CONTEXT || TBB_JOIN_OUTER_TASK_GROUP - start_for& a = *new(task::allocate_root()) start_for(range,body,partitioner); -#else - // Bound context prevents exceptions from body to affect nesting or sibling algorithms, - // and allows users to handle exceptions safely by wrapping parallel_for in the try-block. - task_group_context context(PARALLEL_FOR); - start_for& a = *new(task::allocate_root(context)) start_for(range,body,partitioner); -#endif /* __TBB_TASK_GROUP_CONTEXT && !TBB_JOIN_OUTER_TASK_GROUP */ - // REGION BEGIN - fgt_begin_algorithm( tbb::internal::PARALLEL_FOR_TASK, (void*)&context ); - task::spawn_root_and_wait(a); - fgt_end_algorithm( (void*)&context ); - // REGION END - } - } -#if __TBB_TASK_GROUP_CONTEXT - static void run( const Range& range, const Body& body, Partitioner& partitioner, task_group_context& context ) { - if( !range.empty() ) { - start_for& a = *new(task::allocate_root(context)) start_for(range,body,partitioner); - // REGION BEGIN - fgt_begin_algorithm( tbb::internal::PARALLEL_FOR_TASK, (void*)&context ); - task::spawn_root_and_wait(a); - fgt_end_algorithm( (void*)&context ); - // END REGION - } - } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - //! Run body for range, serves as callback for partitioner - void run_body( Range &r ) { - fgt_alg_begin_body( tbb::internal::PARALLEL_FOR_TASK, (void *)const_cast(&(this->my_body)), (void*)this ); - my_body( r ); - fgt_alg_end_body( (void *)const_cast(&(this->my_body)) ); - } - - //! spawn right task, serves as callback for partitioner - void offer_work(typename Partitioner::split_type& split_obj) { - spawn( *new( allocate_sibling(static_cast(this), sizeof(start_for)) ) start_for(*this, split_obj) ); - } - //! spawn right task, serves as callback for partitioner - void offer_work(const Range& r, depth_t d = 0) { - spawn( *new( allocate_sibling(static_cast(this), sizeof(start_for)) ) start_for(*this, r, d) ); - } - }; - - //! allocate right task with new parent - // TODO: 'inline' here is to avoid multiple definition error but for sake of code size this should not be inlined - inline void* allocate_sibling(task* start_for_task, size_t bytes) { - task* parent_ptr = new( start_for_task->allocate_continuation() ) flag_task(); - start_for_task->set_parent(parent_ptr); - parent_ptr->set_ref_count(2); - return &parent_ptr->allocate_child().allocate(bytes); - } - - //! execute task for parallel_for - template - task* start_for::execute() { - my_partition.check_being_stolen( *this ); - my_partition.execute(*this, my_range); - return NULL; - } -} // namespace internal -//! @endcond -} // namespace interfaceX - -//! @cond INTERNAL -namespace internal { - using interface9::internal::start_for; - - //! Calls the function with values from range [begin, end) with a step provided - template - class parallel_for_body : internal::no_assign { - const Function &my_func; - const Index my_begin; - const Index my_step; - public: - parallel_for_body( const Function& _func, Index& _begin, Index& _step ) - : my_func(_func), my_begin(_begin), my_step(_step) {} - - void operator()( const tbb::blocked_range& r ) const { - // A set of local variables to help the compiler with vectorization of the following loop. - Index b = r.begin(); - Index e = r.end(); - Index ms = my_step; - Index k = my_begin + b*ms; - -#if __INTEL_COMPILER -#pragma ivdep -#if __TBB_ASSERT_ON_VECTORIZATION_FAILURE -#pragma vector always assert -#endif -#endif - for ( Index i = b; i < e; ++i, k += ms ) { - my_func( k ); - } - } - }; -} // namespace internal -//! @endcond - -// Requirements on Range concept are documented in blocked_range.h - -/** \page parallel_for_body_req Requirements on parallel_for body - Class \c Body implementing the concept of parallel_for body must define: - - \code Body::Body( const Body& ); \endcode Copy constructor - - \code Body::~Body(); \endcode Destructor - - \code void Body::operator()( Range& r ) const; \endcode Function call operator applying the body to range \c r. -**/ - -/** \name parallel_for - See also requirements on \ref range_req "Range" and \ref parallel_for_body_req "parallel_for Body". **/ -//@{ - -//! Parallel iteration over range with default partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body ) { - internal::start_for::run(range,body,__TBB_DEFAULT_PARTITIONER()); -} - -//! Parallel iteration over range with simple partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const simple_partitioner& partitioner ) { - internal::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with auto_partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const auto_partitioner& partitioner ) { - internal::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with static_partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const static_partitioner& partitioner ) { - internal::start_for::run(range,body,partitioner); -} - -//! Parallel iteration over range with affinity_partitioner. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, affinity_partitioner& partitioner ) { - internal::start_for::run(range,body,partitioner); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration over range with default partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, task_group_context& context ) { - internal::start_for::run(range, body, __TBB_DEFAULT_PARTITIONER(), context); -} - -//! Parallel iteration over range with simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const simple_partitioner& partitioner, task_group_context& context ) { - internal::start_for::run(range, body, partitioner, context); -} - -//! Parallel iteration over range with auto_partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const auto_partitioner& partitioner, task_group_context& context ) { - internal::start_for::run(range, body, partitioner, context); -} - -//! Parallel iteration over range with static_partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, const static_partitioner& partitioner, task_group_context& context ) { - internal::start_for::run(range, body, partitioner, context); -} - -//! Parallel iteration over range with affinity_partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_for( const Range& range, const Body& body, affinity_partitioner& partitioner, task_group_context& context ) { - internal::start_for::run(range,body,partitioner, context); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ -//@} - -namespace strict_ppl { - -//@{ -//! Implementation of parallel iteration over stepped range of integers with explicit step and partitioner -template -void parallel_for_impl(Index first, Index last, Index step, const Function& f, Partitioner& partitioner) { - if (step <= 0 ) - internal::throw_exception(internal::eid_nonpositive_step); // throws std::invalid_argument - else if (last > first) { - // Above "else" avoids "potential divide by zero" warning on some platforms - Index end = (last - first - Index(1)) / step + Index(1); - tbb::blocked_range range(static_cast(0), end); - internal::parallel_for_body body(f, first, step); - tbb::parallel_for(range, body, partitioner); - } -} - -//! Parallel iteration over a range of integers with a step provided and default partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f) { - parallel_for_impl(first, last, step, f, auto_partitioner()); -} -//! Parallel iteration over a range of integers with a step provided and simple partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const simple_partitioner& partitioner) { - parallel_for_impl(first, last, step, f, partitioner); -} -//! Parallel iteration over a range of integers with a step provided and auto partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const auto_partitioner& partitioner) { - parallel_for_impl(first, last, step, f, partitioner); -} -//! Parallel iteration over a range of integers with a step provided and static partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const static_partitioner& partitioner) { - parallel_for_impl(first, last, step, f, partitioner); -} -//! Parallel iteration over a range of integers with a step provided and affinity partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, affinity_partitioner& partitioner) { - parallel_for_impl(first, last, step, f, partitioner); -} - -//! Parallel iteration over a range of integers with a default step value and default partitioner -template -void parallel_for(Index first, Index last, const Function& f) { - parallel_for_impl(first, last, static_cast(1), f, auto_partitioner()); -} -//! Parallel iteration over a range of integers with a default step value and simple partitioner -template -void parallel_for(Index first, Index last, const Function& f, const simple_partitioner& partitioner) { - parallel_for_impl(first, last, static_cast(1), f, partitioner); -} -//! Parallel iteration over a range of integers with a default step value and auto partitioner -template -void parallel_for(Index first, Index last, const Function& f, const auto_partitioner& partitioner) { - parallel_for_impl(first, last, static_cast(1), f, partitioner); -} -//! Parallel iteration over a range of integers with a default step value and static partitioner -template -void parallel_for(Index first, Index last, const Function& f, const static_partitioner& partitioner) { - parallel_for_impl(first, last, static_cast(1), f, partitioner); -} -//! Parallel iteration over a range of integers with a default step value and affinity partitioner -template -void parallel_for(Index first, Index last, const Function& f, affinity_partitioner& partitioner) { - parallel_for_impl(first, last, static_cast(1), f, partitioner); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Implementation of parallel iteration over stepped range of integers with explicit step, task group context, and partitioner -template -void parallel_for_impl(Index first, Index last, Index step, const Function& f, Partitioner& partitioner, tbb::task_group_context &context) { - if (step <= 0 ) - internal::throw_exception(internal::eid_nonpositive_step); // throws std::invalid_argument - else if (last > first) { - // Above "else" avoids "potential divide by zero" warning on some platforms - Index end = (last - first - Index(1)) / step + Index(1); - tbb::blocked_range range(static_cast(0), end); - internal::parallel_for_body body(f, first, step); - tbb::parallel_for(range, body, partitioner, context); - } -} - -//! Parallel iteration over a range of integers with explicit step, task group context, and default partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, tbb::task_group_context &context) { - parallel_for_impl(first, last, step, f, auto_partitioner(), context); -} -//! Parallel iteration over a range of integers with explicit step, task group context, and simple partitioner - template -void parallel_for(Index first, Index last, Index step, const Function& f, const simple_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, step, f, partitioner, context); -} -//! Parallel iteration over a range of integers with explicit step, task group context, and auto partitioner - template -void parallel_for(Index first, Index last, Index step, const Function& f, const auto_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, step, f, partitioner, context); -} -//! Parallel iteration over a range of integers with explicit step, task group context, and static partitioner -template -void parallel_for(Index first, Index last, Index step, const Function& f, const static_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, step, f, partitioner, context); -} -//! Parallel iteration over a range of integers with explicit step, task group context, and affinity partitioner - template -void parallel_for(Index first, Index last, Index step, const Function& f, affinity_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, step, f, partitioner, context); -} - - -//! Parallel iteration over a range of integers with a default step value, explicit task group context, and default partitioner -template -void parallel_for(Index first, Index last, const Function& f, tbb::task_group_context &context) { - parallel_for_impl(first, last, static_cast(1), f, auto_partitioner(), context); -} -//! Parallel iteration over a range of integers with a default step value, explicit task group context, and simple partitioner - template -void parallel_for(Index first, Index last, const Function& f, const simple_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, static_cast(1), f, partitioner, context); -} -//! Parallel iteration over a range of integers with a default step value, explicit task group context, and auto partitioner - template -void parallel_for(Index first, Index last, const Function& f, const auto_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, static_cast(1), f, partitioner, context); -} -//! Parallel iteration over a range of integers with a default step value, explicit task group context, and static partitioner -template -void parallel_for(Index first, Index last, const Function& f, const static_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, static_cast(1), f, partitioner, context); -} -//! Parallel iteration over a range of integers with a default step value, explicit task group context, and affinity_partitioner - template -void parallel_for(Index first, Index last, const Function& f, affinity_partitioner& partitioner, tbb::task_group_context &context) { - parallel_for_impl(first, last, static_cast(1), f, partitioner, context); -} - -#endif /* __TBB_TASK_GROUP_CONTEXT */ -//@} - -} // namespace strict_ppl - -using strict_ppl::parallel_for; - -} // namespace tbb - -#if TBB_PREVIEW_SERIAL_SUBSET -#define __TBB_NORMAL_EXECUTION -#include "../serial/tbb/parallel_for.h" -#undef __TBB_NORMAL_EXECUTION -#endif - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_parallel_for_H_include_area - -#endif /* __TBB_parallel_for_H */ +#include "../oneapi/tbb/parallel_for.h" diff --git a/src/3rd party/tbb/parallel_for_each.h b/src/3rd party/tbb/parallel_for_each.h index 1ce39ece9d..27c2ab1727 100644 --- a/src/3rd party/tbb/parallel_for_each.h +++ b/src/3rd party/tbb/parallel_for_each.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,120 +14,4 @@ limitations under the License. */ -#ifndef __TBB_parallel_for_each_H -#define __TBB_parallel_for_each_H - -#include "parallel_do.h" -#include "parallel_for.h" - -namespace tbb { - -//! @cond INTERNAL -namespace internal { - // The class calls user function in operator() - template - class parallel_for_each_body_do : internal::no_assign { - const Function &my_func; - public: - parallel_for_each_body_do(const Function &_func) : my_func(_func) {} - - void operator()(typename std::iterator_traits::reference value) const { - my_func(value); - } - }; - - // The class calls user function in operator() - template - class parallel_for_each_body_for : internal::no_assign { - const Function &my_func; - public: - parallel_for_each_body_for(const Function &_func) : my_func(_func) {} - - void operator()(tbb::blocked_range range) const { -#if __INTEL_COMPILER -#pragma ivdep -#endif - for(Iterator it = range.begin(), end = range.end(); it != end; ++it) { - my_func(*it); - } - } - }; - - template - struct parallel_for_each_impl { -#if __TBB_TASK_GROUP_CONTEXT - static void doit(Iterator first, Iterator last, const Function& f, task_group_context &context) { - internal::parallel_for_each_body_do body(f); - tbb::parallel_do(first, last, body, context); - } -#endif - static void doit(Iterator first, Iterator last, const Function& f) { - internal::parallel_for_each_body_do body(f); - tbb::parallel_do(first, last, body); - } - }; - template - struct parallel_for_each_impl { -#if __TBB_TASK_GROUP_CONTEXT - static void doit(Iterator first, Iterator last, const Function& f, task_group_context &context) { - internal::parallel_for_each_body_for body(f); - tbb::parallel_for(tbb::blocked_range(first, last), body, context); - } -#endif - static void doit(Iterator first, Iterator last, const Function& f) { - internal::parallel_for_each_body_for body(f); - tbb::parallel_for(tbb::blocked_range(first, last), body); - } - }; -} // namespace internal -//! @endcond - -/** \name parallel_for_each - **/ -//@{ -//! Calls function f for all items from [first, last) interval using user-supplied context -/** @ingroup algorithms */ -#if __TBB_TASK_GROUP_CONTEXT -template -void parallel_for_each(Iterator first, Iterator last, const Function& f, task_group_context &context) { - internal::parallel_for_each_impl::iterator_category>::doit(first, last, f, context); -} - -//! Calls function f for all items from rng using user-supplied context -/** @ingroup algorithms */ -template -void parallel_for_each(Range& rng, const Function& f, task_group_context& context) { - parallel_for_each(tbb::internal::first(rng), tbb::internal::last(rng), f, context); -} - -//! Calls function f for all items from const rng user-supplied context -/** @ingroup algorithms */ -template -void parallel_for_each(const Range& rng, const Function& f, task_group_context& context) { - parallel_for_each(tbb::internal::first(rng), tbb::internal::last(rng), f, context); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -//! Uses default context -template -void parallel_for_each(Iterator first, Iterator last, const Function& f) { - internal::parallel_for_each_impl::iterator_category>::doit(first, last, f); -} - -//! Uses default context -template -void parallel_for_each(Range& rng, const Function& f) { - parallel_for_each(tbb::internal::first(rng), tbb::internal::last(rng), f); -} - -//! Uses default context -template -void parallel_for_each(const Range& rng, const Function& f) { - parallel_for_each(tbb::internal::first(rng), tbb::internal::last(rng), f); -} - -//@} - -} // namespace - -#endif /* __TBB_parallel_for_each_H */ +#include "../oneapi/tbb/parallel_for_each.h" diff --git a/src/3rd party/tbb/parallel_invoke.h b/src/3rd party/tbb/parallel_invoke.h index 0dffc8108d..6c21100e70 100644 --- a/src/3rd party/tbb/parallel_invoke.h +++ b/src/3rd party/tbb/parallel_invoke.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,447 +14,4 @@ limitations under the License. */ -#ifndef __TBB_parallel_invoke_H -#define __TBB_parallel_invoke_H - -#define __TBB_parallel_invoke_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "task.h" -#include "tbb_profiling.h" - -#if __TBB_VARIADIC_PARALLEL_INVOKE - #include // std::forward -#endif - -namespace tbb { - -#if !__TBB_TASK_GROUP_CONTEXT - /** Dummy to avoid cluttering the bulk of the header with enormous amount of ifdefs. **/ - struct task_group_context { - task_group_context(tbb::internal::string_index){} - }; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -//! @cond INTERNAL -namespace internal { - // Simple task object, executing user method - template - class function_invoker : public task{ - public: - function_invoker(const function& _function) : my_function(_function) {} - private: - const function &my_function; - task* execute() __TBB_override - { - my_function(); - return NULL; - } - }; - - // The class spawns two or three child tasks - template - class spawner : public task { - private: - const function1& my_func1; - const function2& my_func2; - const function3& my_func3; - bool is_recycled; - - task* execute () __TBB_override { - if(is_recycled){ - return NULL; - }else{ - __TBB_ASSERT(N==2 || N==3, "Number of arguments passed to spawner is wrong"); - set_ref_count(N); - recycle_as_safe_continuation(); - internal::function_invoker* invoker2 = new (allocate_child()) internal::function_invoker(my_func2); - __TBB_ASSERT(invoker2, "Child task allocation failed"); - spawn(*invoker2); - size_t n = N; // To prevent compiler warnings - if (n>2) { - internal::function_invoker* invoker3 = new (allocate_child()) internal::function_invoker(my_func3); - __TBB_ASSERT(invoker3, "Child task allocation failed"); - spawn(*invoker3); - } - my_func1(); - is_recycled = true; - return NULL; - } - } // execute - - public: - spawner(const function1& _func1, const function2& _func2, const function3& _func3) : my_func1(_func1), my_func2(_func2), my_func3(_func3), is_recycled(false) {} - }; - - // Creates and spawns child tasks - class parallel_invoke_helper : public empty_task { - public: - // Dummy functor class - class parallel_invoke_noop { - public: - void operator() () const {} - }; - // Creates a helper object with user-defined number of children expected - parallel_invoke_helper(int number_of_children) - { - set_ref_count(number_of_children + 1); - } - -#if __TBB_VARIADIC_PARALLEL_INVOKE - void add_children() {} - void add_children(tbb::task_group_context&) {} - - template - void add_children(function&& _func) - { - internal::function_invoker* invoker = new (allocate_child()) internal::function_invoker(std::forward(_func)); - __TBB_ASSERT(invoker, "Child task allocation failed"); - spawn(*invoker); - } - - template - void add_children(function&& _func, tbb::task_group_context&) - { - add_children(std::forward(_func)); - } - - // Adds child(ren) task(s) and spawns them - template - void add_children(function1&& _func1, function2&& _func2, function&&... _func) - { - // The third argument is dummy, it is ignored actually. - parallel_invoke_noop noop; - typedef internal::spawner<2, function1, function2, parallel_invoke_noop> spawner_type; - spawner_type & sub_root = *new(allocate_child()) spawner_type(std::forward(_func1), std::forward(_func2), noop); - spawn(sub_root); - add_children(std::forward(_func)...); - } -#else - // Adds child task and spawns it - template - void add_children (const function &_func) - { - internal::function_invoker* invoker = new (allocate_child()) internal::function_invoker(_func); - __TBB_ASSERT(invoker, "Child task allocation failed"); - spawn(*invoker); - } - - // Adds a task with multiple child tasks and spawns it - // two arguments - template - void add_children (const function1& _func1, const function2& _func2) - { - // The third argument is dummy, it is ignored actually. - parallel_invoke_noop noop; - internal::spawner<2, function1, function2, parallel_invoke_noop>& sub_root = *new(allocate_child())internal::spawner<2, function1, function2, parallel_invoke_noop>(_func1, _func2, noop); - spawn(sub_root); - } - // three arguments - template - void add_children (const function1& _func1, const function2& _func2, const function3& _func3) - { - internal::spawner<3, function1, function2, function3>& sub_root = *new(allocate_child())internal::spawner<3, function1, function2, function3>(_func1, _func2, _func3); - spawn(sub_root); - } -#endif // __TBB_VARIADIC_PARALLEL_INVOKE - - // Waits for all child tasks - template - void run_and_finish(const F0& f0) - { - internal::function_invoker* invoker = new (allocate_child()) internal::function_invoker(f0); - __TBB_ASSERT(invoker, "Child task allocation failed"); - spawn_and_wait_for_all(*invoker); - } - }; - // The class destroys root if exception occurred as well as in normal case - class parallel_invoke_cleaner: internal::no_copy { - public: -#if __TBB_TASK_GROUP_CONTEXT - parallel_invoke_cleaner(int number_of_children, tbb::task_group_context& context) - : root(*new(task::allocate_root(context)) internal::parallel_invoke_helper(number_of_children)) -#else - parallel_invoke_cleaner(int number_of_children, tbb::task_group_context&) - : root(*new(task::allocate_root()) internal::parallel_invoke_helper(number_of_children)) -#endif /* !__TBB_TASK_GROUP_CONTEXT */ - {} - - ~parallel_invoke_cleaner(){ - root.destroy(root); - } - internal::parallel_invoke_helper& root; - }; - -#if __TBB_VARIADIC_PARALLEL_INVOKE -// Determine whether the last parameter in a pack is task_group_context - template struct impl_selector; // to workaround a GCC bug - - template struct impl_selector { - typedef typename impl_selector::type type; - }; - - template struct impl_selector { - typedef false_type type; - }; - template<> struct impl_selector { - typedef true_type type; - }; - - // Select task_group_context parameter from the back of a pack - inline task_group_context& get_context( task_group_context& tgc ) { return tgc; } - - template - task_group_context& get_context( T1&& /*ignored*/, T&&... t ) - { return get_context( std::forward(t)... ); } - - // task_group_context is known to be at the back of the parameter pack - template - void parallel_invoke_impl(true_type, F0&& f0, F1&& f1, F&&... f) { - __TBB_STATIC_ASSERT(sizeof...(F)>0, "Variadic parallel_invoke implementation broken?"); - // # of child tasks: f0, f1, and a task for each two elements of the pack except the last - const size_t number_of_children = 2 + sizeof...(F)/2; - parallel_invoke_cleaner cleaner(number_of_children, get_context(std::forward(f)...)); - parallel_invoke_helper& root = cleaner.root; - - root.add_children(std::forward(f)...); - root.add_children(std::forward(f1)); - root.run_and_finish(std::forward(f0)); - } - - // task_group_context is not in the pack, needs to be added - template - void parallel_invoke_impl(false_type, F0&& f0, F1&& f1, F&&... f) { - tbb::task_group_context context(PARALLEL_INVOKE); - // Add context to the arguments, and redirect to the other overload - parallel_invoke_impl(true_type(), std::forward(f0), std::forward(f1), std::forward(f)..., context); - } -#endif -} // namespace internal -//! @endcond - -/** \name parallel_invoke - **/ -//@{ -//! Executes a list of tasks in parallel and waits for all tasks to complete. -/** @ingroup algorithms */ - -#if __TBB_VARIADIC_PARALLEL_INVOKE - -// parallel_invoke for two or more arguments via variadic templates -// presence of task_group_context is defined automatically -template -void parallel_invoke(F0&& f0, F1&& f1, F&&... f) { - typedef typename internal::impl_selector::type selector_type; - internal::parallel_invoke_impl(selector_type(), std::forward(f0), std::forward(f1), std::forward(f)...); -} - -#else - -// parallel_invoke with user-defined context -// two arguments -template -void parallel_invoke(const F0& f0, const F1& f1, tbb::task_group_context& context) { - internal::parallel_invoke_cleaner cleaner(2, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f1); - - root.run_and_finish(f0); -} - -// three arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, tbb::task_group_context& context) { - internal::parallel_invoke_cleaner cleaner(3, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f2); - root.add_children(f1); - - root.run_and_finish(f0); -} - -// four arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(4, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f3); - root.add_children(f2); - root.add_children(f1); - - root.run_and_finish(f0); -} - -// five arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(3, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f4, f3); - root.add_children(f2, f1); - - root.run_and_finish(f0); -} - -// six arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, const F5& f5, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(3, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f5, f4, f3); - root.add_children(f2, f1); - - root.run_and_finish(f0); -} - -// seven arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(3, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f6, f5, f4); - root.add_children(f3, f2, f1); - - root.run_and_finish(f0); -} - -// eight arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(4, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f7, f6, f5); - root.add_children(f4, f3); - root.add_children(f2, f1); - - root.run_and_finish(f0); -} - -// nine arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7, const F8& f8, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(4, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f8, f7, f6); - root.add_children(f5, f4, f3); - root.add_children(f2, f1); - - root.run_and_finish(f0); -} - -// ten arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7, const F8& f8, const F9& f9, - tbb::task_group_context& context) -{ - internal::parallel_invoke_cleaner cleaner(4, context); - internal::parallel_invoke_helper& root = cleaner.root; - - root.add_children(f9, f8, f7); - root.add_children(f6, f5, f4); - root.add_children(f3, f2, f1); - - root.run_and_finish(f0); -} - -// two arguments -template -void parallel_invoke(const F0& f0, const F1& f1) { - task_group_context context(internal::PARALLEL_INVOKE); - parallel_invoke(f0, f1, context); -} -// three arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2) { - task_group_context context(internal::PARALLEL_INVOKE); - parallel_invoke(f0, f1, f2, context); -} -// four arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3) { - task_group_context context(internal::PARALLEL_INVOKE); - parallel_invoke(f0, f1, f2, f3, context); -} -// five arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4) { - task_group_context context(internal::PARALLEL_INVOKE); - parallel_invoke(f0, f1, f2, f3, f4, context); -} -// six arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, const F5& f5) { - task_group_context context(internal::PARALLEL_INVOKE); - parallel_invoke(f0, f1, f2, f3, f4, f5, context); -} -// seven arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6) -{ - task_group_context context(internal::PARALLEL_INVOKE); - parallel_invoke(f0, f1, f2, f3, f4, f5, f6, context); -} -// eight arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7) -{ - task_group_context context(internal::PARALLEL_INVOKE); - parallel_invoke(f0, f1, f2, f3, f4, f5, f6, f7, context); -} -// nine arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7, const F8& f8) -{ - task_group_context context(internal::PARALLEL_INVOKE); - parallel_invoke(f0, f1, f2, f3, f4, f5, f6, f7, f8, context); -} -// ten arguments -template -void parallel_invoke(const F0& f0, const F1& f1, const F2& f2, const F3& f3, const F4& f4, - const F5& f5, const F6& f6, const F7& f7, const F8& f8, const F9& f9) -{ - task_group_context context(internal::PARALLEL_INVOKE); - parallel_invoke(f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, context); -} -#endif // __TBB_VARIADIC_PARALLEL_INVOKE -//@} - -} // namespace - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_parallel_invoke_H_include_area - -#endif /* __TBB_parallel_invoke_H */ +#include "../oneapi/tbb/parallel_invoke.h" diff --git a/src/3rd party/tbb/parallel_pipeline.h b/src/3rd party/tbb/parallel_pipeline.h new file mode 100644 index 0000000000..aceee49f8a --- /dev/null +++ b/src/3rd party/tbb/parallel_pipeline.h @@ -0,0 +1,17 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "../oneapi/tbb/parallel_pipeline.h" diff --git a/src/3rd party/tbb/parallel_reduce.h b/src/3rd party/tbb/parallel_reduce.h index 24e36cbd46..83658755a8 100644 --- a/src/3rd party/tbb/parallel_reduce.h +++ b/src/3rd party/tbb/parallel_reduce.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,644 +14,4 @@ limitations under the License. */ -#ifndef __TBB_parallel_reduce_H -#define __TBB_parallel_reduce_H - -#define __TBB_parallel_reduce_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include -#include "task.h" -#include "aligned_space.h" -#include "partitioner.h" -#include "tbb_profiling.h" - -namespace tbb { - -namespace interface9 { -//! @cond INTERNAL -namespace internal { - - using namespace tbb::internal; - - /** Values for reduction_context. */ - enum { - root_task, left_child, right_child - }; - - /** Represented as a char, not enum, for compactness. */ - typedef char reduction_context; - - //! Task type used to combine the partial results of parallel_reduce. - /** @ingroup algorithms */ - template - class finish_reduce: public flag_task { - //! Pointer to body, or NULL if the left child has not yet finished. - bool has_right_zombie; - const reduction_context my_context; - Body* my_body; - aligned_space zombie_space; - finish_reduce( reduction_context context_ ) : - has_right_zombie(false), // TODO: substitute by flag_task::child_stolen? - my_context(context_), - my_body(NULL) - { - } - ~finish_reduce() { - if( has_right_zombie ) - zombie_space.begin()->~Body(); - } - task* execute() __TBB_override { - if( has_right_zombie ) { - // Right child was stolen. - Body* s = zombie_space.begin(); - my_body->join( *s ); - // Body::join() won't be called if canceled. Defer destruction to destructor - } - if( my_context==left_child ) - itt_store_word_with_release( static_cast(parent())->my_body, my_body ); - return NULL; - } - template - friend class start_reduce; - }; - - //! allocate right task with new parent - void allocate_sibling(task* start_reduce_task, task *tasks[], size_t start_bytes, size_t finish_bytes); - - //! Task type used to split the work of parallel_reduce. - /** @ingroup algorithms */ - template - class start_reduce: public task { - typedef finish_reduce finish_type; - Body* my_body; - Range my_range; - typename Partitioner::task_partition_type my_partition; - reduction_context my_context; - task* execute() __TBB_override; - //! Update affinity info, if any - void note_affinity( affinity_id id ) __TBB_override { - my_partition.note_affinity( id ); - } - template - friend class finish_reduce; - -public: - //! Constructor used for root task - start_reduce( const Range& range, Body* body, Partitioner& partitioner ) : - my_body(body), - my_range(range), - my_partition(partitioner), - my_context(root_task) - { - } - //! Splitting constructor used to generate children. - /** parent_ becomes left child. Newly constructed object is right child. */ - start_reduce( start_reduce& parent_, typename Partitioner::split_type& split_obj ) : - my_body(parent_.my_body), - my_range(parent_.my_range, split_obj), - my_partition(parent_.my_partition, split_obj), - my_context(right_child) - { - my_partition.set_affinity(*this); - parent_.my_context = left_child; - } - //! Construct right child from the given range as response to the demand. - /** parent_ remains left child. Newly constructed object is right child. */ - start_reduce( start_reduce& parent_, const Range& r, depth_t d ) : - my_body(parent_.my_body), - my_range(r), - my_partition(parent_.my_partition, split()), - my_context(right_child) - { - my_partition.set_affinity(*this); - my_partition.align_depth( d ); // TODO: move into constructor of partitioner - parent_.my_context = left_child; - } - static void run( const Range& range, Body& body, Partitioner& partitioner ) { - if( !range.empty() ) { -#if !__TBB_TASK_GROUP_CONTEXT || TBB_JOIN_OUTER_TASK_GROUP - task::spawn_root_and_wait( *new(task::allocate_root()) start_reduce(range,&body,partitioner) ); -#else - // Bound context prevents exceptions from body to affect nesting or sibling algorithms, - // and allows users to handle exceptions safely by wrapping parallel_for in the try-block. - task_group_context context(PARALLEL_REDUCE); - task::spawn_root_and_wait( *new(task::allocate_root(context)) start_reduce(range,&body,partitioner) ); -#endif /* __TBB_TASK_GROUP_CONTEXT && !TBB_JOIN_OUTER_TASK_GROUP */ - } - } -#if __TBB_TASK_GROUP_CONTEXT - static void run( const Range& range, Body& body, Partitioner& partitioner, task_group_context& context ) { - if( !range.empty() ) - task::spawn_root_and_wait( *new(task::allocate_root(context)) start_reduce(range,&body,partitioner) ); - } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - //! Run body for range - void run_body( Range &r ) { (*my_body)( r ); } - - //! spawn right task, serves as callback for partitioner - // TODO: remove code duplication from 'offer_work' methods - void offer_work(typename Partitioner::split_type& split_obj) { - task *tasks[2]; - allocate_sibling(static_cast(this), tasks, sizeof(start_reduce), sizeof(finish_type)); - new((void*)tasks[0]) finish_type(my_context); - new((void*)tasks[1]) start_reduce(*this, split_obj); - spawn(*tasks[1]); - } - //! spawn right task, serves as callback for partitioner - void offer_work(const Range& r, depth_t d = 0) { - task *tasks[2]; - allocate_sibling(static_cast(this), tasks, sizeof(start_reduce), sizeof(finish_type)); - new((void*)tasks[0]) finish_type(my_context); - new((void*)tasks[1]) start_reduce(*this, r, d); - spawn(*tasks[1]); - } - }; - - //! allocate right task with new parent - // TODO: 'inline' here is to avoid multiple definition error but for sake of code size this should not be inlined - inline void allocate_sibling(task* start_reduce_task, task *tasks[], size_t start_bytes, size_t finish_bytes) { - tasks[0] = &start_reduce_task->allocate_continuation().allocate(finish_bytes); - start_reduce_task->set_parent(tasks[0]); - tasks[0]->set_ref_count(2); - tasks[1] = &tasks[0]->allocate_child().allocate(start_bytes); - } - - template - task* start_reduce::execute() { - my_partition.check_being_stolen( *this ); - if( my_context==right_child ) { - finish_type* parent_ptr = static_cast(parent()); - if( !itt_load_word_with_acquire(parent_ptr->my_body) ) { // TODO: replace by is_stolen_task() or by parent_ptr->ref_count() == 2??? - my_body = new( parent_ptr->zombie_space.begin() ) Body(*my_body,split()); - parent_ptr->has_right_zombie = true; - } - } else __TBB_ASSERT(my_context==root_task,NULL);// because left leaf spawns right leafs without recycling - my_partition.execute(*this, my_range); - if( my_context==left_child ) { - finish_type* parent_ptr = static_cast(parent()); - __TBB_ASSERT(my_body!=parent_ptr->zombie_space.begin(),NULL); - itt_store_word_with_release(parent_ptr->my_body, my_body ); - } - return NULL; - } - - //! Task type used to combine the partial results of parallel_deterministic_reduce. - /** @ingroup algorithms */ - template - class finish_deterministic_reduce: public task { - Body &my_left_body; - Body my_right_body; - - finish_deterministic_reduce( Body &body ) : - my_left_body( body ), - my_right_body( body, split() ) - { - } - task* execute() __TBB_override { - my_left_body.join( my_right_body ); - return NULL; - } - template - friend class start_deterministic_reduce; - }; - - //! Task type used to split the work of parallel_deterministic_reduce. - /** @ingroup algorithms */ - template - class start_deterministic_reduce: public task { - typedef finish_deterministic_reduce finish_type; - Body &my_body; - Range my_range; - typename Partitioner::task_partition_type my_partition; - task* execute() __TBB_override; - - //! Constructor used for root task - start_deterministic_reduce( const Range& range, Body& body, Partitioner& partitioner ) : - my_body( body ), - my_range( range ), - my_partition( partitioner ) - { - } - //! Splitting constructor used to generate children. - /** parent_ becomes left child. Newly constructed object is right child. */ - start_deterministic_reduce( start_deterministic_reduce& parent_, finish_type& c, typename Partitioner::split_type& split_obj ) : - my_body( c.my_right_body ), - my_range( parent_.my_range, split_obj ), - my_partition( parent_.my_partition, split_obj ) - { - } - -public: - static void run( const Range& range, Body& body, Partitioner& partitioner ) { - if( !range.empty() ) { -#if !__TBB_TASK_GROUP_CONTEXT || TBB_JOIN_OUTER_TASK_GROUP - task::spawn_root_and_wait( *new(task::allocate_root()) start_deterministic_reduce(range,&body,partitioner) ); -#else - // Bound context prevents exceptions from body to affect nesting or sibling algorithms, - // and allows users to handle exceptions safely by wrapping parallel_for in the try-block. - task_group_context context(PARALLEL_REDUCE); - task::spawn_root_and_wait( *new(task::allocate_root(context)) start_deterministic_reduce(range,body,partitioner) ); -#endif /* __TBB_TASK_GROUP_CONTEXT && !TBB_JOIN_OUTER_TASK_GROUP */ - } - } -#if __TBB_TASK_GROUP_CONTEXT - static void run( const Range& range, Body& body, Partitioner& partitioner, task_group_context& context ) { - if( !range.empty() ) - task::spawn_root_and_wait( *new(task::allocate_root(context)) start_deterministic_reduce(range,body,partitioner) ); - } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - - void offer_work( typename Partitioner::split_type& split_obj) { - task* tasks[2]; - allocate_sibling(static_cast(this), tasks, sizeof(start_deterministic_reduce), sizeof(finish_type)); - new((void*)tasks[0]) finish_type(my_body); - new((void*)tasks[1]) start_deterministic_reduce(*this, *static_cast(tasks[0]), split_obj); - spawn(*tasks[1]); - } - - void run_body( Range &r ) { my_body(r); } - }; - - template - task* start_deterministic_reduce::execute() { - my_partition.execute(*this, my_range); - return NULL; - } -} // namespace internal -//! @endcond -} //namespace interfaceX - -//! @cond INTERNAL -namespace internal { - using interface9::internal::start_reduce; - using interface9::internal::start_deterministic_reduce; - //! Auxiliary class for parallel_reduce; for internal use only. - /** The adaptor class that implements \ref parallel_reduce_body_req "parallel_reduce Body" - using given \ref parallel_reduce_lambda_req "anonymous function objects". - **/ - /** @ingroup algorithms */ - template - class lambda_reduce_body { - -//FIXME: decide if my_real_body, my_reduction, and identity_element should be copied or referenced -// (might require some performance measurements) - - const Value& identity_element; - const RealBody& my_real_body; - const Reduction& my_reduction; - Value my_value; - lambda_reduce_body& operator= ( const lambda_reduce_body& other ); - public: - lambda_reduce_body( const Value& identity, const RealBody& body, const Reduction& reduction ) - : identity_element(identity) - , my_real_body(body) - , my_reduction(reduction) - , my_value(identity) - { } - lambda_reduce_body( const lambda_reduce_body& other ) - : identity_element(other.identity_element) - , my_real_body(other.my_real_body) - , my_reduction(other.my_reduction) - , my_value(other.my_value) - { } - lambda_reduce_body( lambda_reduce_body& other, tbb::split ) - : identity_element(other.identity_element) - , my_real_body(other.my_real_body) - , my_reduction(other.my_reduction) - , my_value(other.identity_element) - { } - void operator()(Range& range) { - my_value = my_real_body(range, const_cast(my_value)); - } - void join( lambda_reduce_body& rhs ) { - my_value = my_reduction(const_cast(my_value), const_cast(rhs.my_value)); - } - Value result() const { - return my_value; - } - }; - -} // namespace internal -//! @endcond - -// Requirements on Range concept are documented in blocked_range.h - -/** \page parallel_reduce_body_req Requirements on parallel_reduce body - Class \c Body implementing the concept of parallel_reduce body must define: - - \code Body::Body( Body&, split ); \endcode Splitting constructor. - Must be able to run concurrently with operator() and method \c join - - \code Body::~Body(); \endcode Destructor - - \code void Body::operator()( Range& r ); \endcode Function call operator applying body to range \c r - and accumulating the result - - \code void Body::join( Body& b ); \endcode Join results. - The result in \c b should be merged into the result of \c this -**/ - -/** \page parallel_reduce_lambda_req Requirements on parallel_reduce anonymous function objects (lambda functions) - TO BE DOCUMENTED -**/ - -/** \name parallel_reduce - See also requirements on \ref range_req "Range" and \ref parallel_reduce_body_req "parallel_reduce Body". **/ -//@{ - -//! Parallel iteration with reduction and default partitioner. -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body ) { - internal::start_reduce::run( range, body, __TBB_DEFAULT_PARTITIONER() ); -} - -//! Parallel iteration with reduction and simple_partitioner -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const simple_partitioner& partitioner ) { - internal::start_reduce::run( range, body, partitioner ); -} - -//! Parallel iteration with reduction and auto_partitioner -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const auto_partitioner& partitioner ) { - internal::start_reduce::run( range, body, partitioner ); -} - -//! Parallel iteration with reduction and static_partitioner -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const static_partitioner& partitioner ) { - internal::start_reduce::run( range, body, partitioner ); -} - -//! Parallel iteration with reduction and affinity_partitioner -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, affinity_partitioner& partitioner ) { - internal::start_reduce::run( range, body, partitioner ); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration with reduction, default partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, task_group_context& context ) { - internal::start_reduce::run( range, body, __TBB_DEFAULT_PARTITIONER(), context ); -} - -//! Parallel iteration with reduction, simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const simple_partitioner& partitioner, task_group_context& context ) { - internal::start_reduce::run( range, body, partitioner, context ); -} - -//! Parallel iteration with reduction, auto_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const auto_partitioner& partitioner, task_group_context& context ) { - internal::start_reduce::run( range, body, partitioner, context ); -} - -//! Parallel iteration with reduction, static_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, const static_partitioner& partitioner, task_group_context& context ) { - internal::start_reduce::run( range, body, partitioner, context ); -} - -//! Parallel iteration with reduction, affinity_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -void parallel_reduce( const Range& range, Body& body, affinity_partitioner& partitioner, task_group_context& context ) { - internal::start_reduce::run( range, body, partitioner, context ); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -/** parallel_reduce overloads that work with anonymous function objects - (see also \ref parallel_reduce_lambda_req "requirements on parallel_reduce anonymous function objects"). **/ - -//! Parallel iteration with reduction and default partitioner. -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const __TBB_DEFAULT_PARTITIONER> - ::run(range, body, __TBB_DEFAULT_PARTITIONER() ); - return body.result(); -} - -//! Parallel iteration with reduction and simple_partitioner. -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const simple_partitioner& partitioner ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const simple_partitioner> - ::run(range, body, partitioner ); - return body.result(); -} - -//! Parallel iteration with reduction and auto_partitioner -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const auto_partitioner& partitioner ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const auto_partitioner> - ::run( range, body, partitioner ); - return body.result(); -} - -//! Parallel iteration with reduction and static_partitioner -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const static_partitioner& partitioner ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const static_partitioner> - ::run( range, body, partitioner ); - return body.result(); -} - -//! Parallel iteration with reduction and affinity_partitioner -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - affinity_partitioner& partitioner ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,affinity_partitioner> - ::run( range, body, partitioner ); - return body.result(); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration with reduction, default partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const __TBB_DEFAULT_PARTITIONER> - ::run( range, body, __TBB_DEFAULT_PARTITIONER(), context ); - return body.result(); -} - -//! Parallel iteration with reduction, simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const simple_partitioner& partitioner, task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const simple_partitioner> - ::run( range, body, partitioner, context ); - return body.result(); -} - -//! Parallel iteration with reduction, auto_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const auto_partitioner& partitioner, task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const auto_partitioner> - ::run( range, body, partitioner, context ); - return body.result(); -} - -//! Parallel iteration with reduction, static_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const static_partitioner& partitioner, task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,const static_partitioner> - ::run( range, body, partitioner, context ); - return body.result(); -} - -//! Parallel iteration with reduction, affinity_partitioner and user-supplied context -/** @ingroup algorithms **/ -template -Value parallel_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - affinity_partitioner& partitioner, task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_reduce,affinity_partitioner> - ::run( range, body, partitioner, context ); - return body.result(); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -//! Parallel iteration with deterministic reduction and default simple partitioner. -/** @ingroup algorithms **/ -template -void parallel_deterministic_reduce( const Range& range, Body& body ) { - internal::start_deterministic_reduce::run(range, body, simple_partitioner()); -} - -//! Parallel iteration with deterministic reduction and simple partitioner. -/** @ingroup algorithms **/ -template -void parallel_deterministic_reduce( const Range& range, Body& body, const simple_partitioner& partitioner ) { - internal::start_deterministic_reduce::run(range, body, partitioner); -} - -//! Parallel iteration with deterministic reduction and static partitioner. -/** @ingroup algorithms **/ -template -void parallel_deterministic_reduce( const Range& range, Body& body, const static_partitioner& partitioner ) { - internal::start_deterministic_reduce::run(range, body, partitioner); -} - -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration with deterministic reduction, default simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_deterministic_reduce( const Range& range, Body& body, task_group_context& context ) { - internal::start_deterministic_reduce::run( range, body, simple_partitioner(), context ); -} - -//! Parallel iteration with deterministic reduction, simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_deterministic_reduce( const Range& range, Body& body, const simple_partitioner& partitioner, task_group_context& context ) { - internal::start_deterministic_reduce::run(range, body, partitioner, context); -} - -//! Parallel iteration with deterministic reduction, static partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -void parallel_deterministic_reduce( const Range& range, Body& body, const static_partitioner& partitioner, task_group_context& context ) { - internal::start_deterministic_reduce::run(range, body, partitioner, context); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -/** parallel_reduce overloads that work with anonymous function objects - (see also \ref parallel_reduce_lambda_req "requirements on parallel_reduce anonymous function objects"). **/ - -//! Parallel iteration with deterministic reduction and default simple partitioner. -// TODO: consider making static_partitioner the default -/** @ingroup algorithms **/ -template -Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction ) { - return parallel_deterministic_reduce(range, identity, real_body, reduction, simple_partitioner()); -} - -//! Parallel iteration with deterministic reduction and simple partitioner. -/** @ingroup algorithms **/ -template -Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, const simple_partitioner& partitioner ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_deterministic_reduce, const simple_partitioner> - ::run(range, body, partitioner); - return body.result(); -} - -//! Parallel iteration with deterministic reduction and static partitioner. -/** @ingroup algorithms **/ -template -Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, const static_partitioner& partitioner ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_deterministic_reduce, const static_partitioner> - ::run(range, body, partitioner); - return body.result(); -} -#if __TBB_TASK_GROUP_CONTEXT -//! Parallel iteration with deterministic reduction, default simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - task_group_context& context ) { - return parallel_deterministic_reduce(range, identity, real_body, reduction, simple_partitioner(), context); -} - -//! Parallel iteration with deterministic reduction, simple partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const simple_partitioner& partitioner, task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_deterministic_reduce, const simple_partitioner> - ::run(range, body, partitioner, context); - return body.result(); -} - -//! Parallel iteration with deterministic reduction, static partitioner and user-supplied context. -/** @ingroup algorithms **/ -template -Value parallel_deterministic_reduce( const Range& range, const Value& identity, const RealBody& real_body, const Reduction& reduction, - const static_partitioner& partitioner, task_group_context& context ) { - internal::lambda_reduce_body body(identity, real_body, reduction); - internal::start_deterministic_reduce, const static_partitioner> - ::run(range, body, partitioner, context); - return body.result(); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ -//@} - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_parallel_reduce_H_include_area - -#endif /* __TBB_parallel_reduce_H */ +#include "../oneapi/tbb/parallel_reduce.h" diff --git a/src/3rd party/tbb/parallel_scan.h b/src/3rd party/tbb/parallel_scan.h index 302965ea82..682032a561 100644 --- a/src/3rd party/tbb/parallel_scan.h +++ b/src/3rd party/tbb/parallel_scan.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,403 +14,4 @@ limitations under the License. */ -#ifndef __TBB_parallel_scan_H -#define __TBB_parallel_scan_H - -#define __TBB_parallel_scan_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "task.h" -#include "aligned_space.h" -#include -#include "partitioner.h" - -namespace tbb { - -//! Used to indicate that the initial scan is being performed. -/** @ingroup algorithms */ -struct pre_scan_tag { - static bool is_final_scan() {return false;} - operator bool() {return is_final_scan();} -}; - -//! Used to indicate that the final scan is being performed. -/** @ingroup algorithms */ -struct final_scan_tag { - static bool is_final_scan() {return true;} - operator bool() {return is_final_scan();} -}; - -//! @cond INTERNAL -namespace internal { - - //! Performs final scan for a leaf - /** @ingroup algorithms */ - template - class final_sum: public task { - public: - Body my_body; - private: - aligned_space my_range; - //! Where to put result of last subrange, or NULL if not last subrange. - Body* my_stuff_last; - public: - final_sum( Body& body_ ) : - my_body(body_,split()) - { - poison_pointer(my_stuff_last); - } - ~final_sum() { - my_range.begin()->~Range(); - } - void finish_construction( const Range& range_, Body* stuff_last_ ) { - new( my_range.begin() ) Range(range_); - my_stuff_last = stuff_last_; - } - private: - task* execute() __TBB_override { - my_body( *my_range.begin(), final_scan_tag() ); - if( my_stuff_last ) - my_stuff_last->assign(my_body); - return NULL; - } - }; - - //! Split work to be done in the scan. - /** @ingroup algorithms */ - template - class sum_node: public task { - typedef final_sum final_sum_type; - public: - final_sum_type *my_incoming; - final_sum_type *my_body; - Body *my_stuff_last; - private: - final_sum_type *my_left_sum; - sum_node *my_left; - sum_node *my_right; - bool my_left_is_final; - Range my_range; - sum_node( const Range range_, bool left_is_final_ ) : - my_stuff_last(NULL), - my_left_sum(NULL), - my_left(NULL), - my_right(NULL), - my_left_is_final(left_is_final_), - my_range(range_) - { - // Poison fields that will be set by second pass. - poison_pointer(my_body); - poison_pointer(my_incoming); - } - task* create_child( const Range& range_, final_sum_type& f, sum_node* n, final_sum_type* incoming_, Body* stuff_last_ ) { - if( !n ) { - f.recycle_as_child_of( *this ); - f.finish_construction( range_, stuff_last_ ); - return &f; - } else { - n->my_body = &f; - n->my_incoming = incoming_; - n->my_stuff_last = stuff_last_; - return n; - } - } - task* execute() __TBB_override { - if( my_body ) { - if( my_incoming ) - my_left_sum->my_body.reverse_join( my_incoming->my_body ); - recycle_as_continuation(); - sum_node& c = *this; - task* b = c.create_child(Range(my_range,split()),*my_left_sum,my_right,my_left_sum,my_stuff_last); - task* a = my_left_is_final ? NULL : c.create_child(my_range,*my_body,my_left,my_incoming,NULL); - set_ref_count( (a!=NULL)+(b!=NULL) ); - my_body = NULL; - if( a ) spawn(*b); - else a = b; - return a; - } else { - return NULL; - } - } - template - friend class start_scan; - - template - friend class finish_scan; - }; - - //! Combine partial results - /** @ingroup algorithms */ - template - class finish_scan: public task { - typedef sum_node sum_node_type; - typedef final_sum final_sum_type; - final_sum_type** const my_sum; - sum_node_type*& my_return_slot; - public: - final_sum_type* my_right_zombie; - sum_node_type& my_result; - - task* execute() __TBB_override { - __TBB_ASSERT( my_result.ref_count()==(my_result.my_left!=NULL)+(my_result.my_right!=NULL), NULL ); - if( my_result.my_left ) - my_result.my_left_is_final = false; - if( my_right_zombie && my_sum ) - ((*my_sum)->my_body).reverse_join(my_result.my_left_sum->my_body); - __TBB_ASSERT( !my_return_slot, NULL ); - if( my_right_zombie || my_result.my_right ) { - my_return_slot = &my_result; - } else { - destroy( my_result ); - } - if( my_right_zombie && !my_sum && !my_result.my_right ) { - destroy(*my_right_zombie); - my_right_zombie = NULL; - } - return NULL; - } - - finish_scan( sum_node_type*& return_slot_, final_sum_type** sum_, sum_node_type& result_ ) : - my_sum(sum_), - my_return_slot(return_slot_), - my_right_zombie(NULL), - my_result(result_) - { - __TBB_ASSERT( !my_return_slot, NULL ); - } - }; - - //! Initial task to split the work - /** @ingroup algorithms */ - template - class start_scan: public task { - typedef sum_node sum_node_type; - typedef final_sum final_sum_type; - final_sum_type* my_body; - /** Non-null if caller is requesting total. */ - final_sum_type** my_sum; - sum_node_type** my_return_slot; - /** Null if computing root. */ - sum_node_type* my_parent_sum; - bool my_is_final; - bool my_is_right_child; - Range my_range; - typename Partitioner::partition_type my_partition; - task* execute() __TBB_override ; - public: - start_scan( sum_node_type*& return_slot_, start_scan& parent_, sum_node_type* parent_sum_ ) : - my_body(parent_.my_body), - my_sum(parent_.my_sum), - my_return_slot(&return_slot_), - my_parent_sum(parent_sum_), - my_is_final(parent_.my_is_final), - my_is_right_child(false), - my_range(parent_.my_range,split()), - my_partition(parent_.my_partition,split()) - { - __TBB_ASSERT( !*my_return_slot, NULL ); - } - - start_scan( sum_node_type*& return_slot_, const Range& range_, final_sum_type& body_, const Partitioner& partitioner_) : - my_body(&body_), - my_sum(NULL), - my_return_slot(&return_slot_), - my_parent_sum(NULL), - my_is_final(true), - my_is_right_child(false), - my_range(range_), - my_partition(partitioner_) - { - __TBB_ASSERT( !*my_return_slot, NULL ); - } - - static void run( const Range& range_, Body& body_, const Partitioner& partitioner_ ) { - if( !range_.empty() ) { - typedef internal::start_scan start_pass1_type; - internal::sum_node* root = NULL; - final_sum_type* temp_body = new(task::allocate_root()) final_sum_type( body_ ); - start_pass1_type& pass1 = *new(task::allocate_root()) start_pass1_type( - /*my_return_slot=*/root, - range_, - *temp_body, - partitioner_ ); - temp_body->my_body.reverse_join(body_); - task::spawn_root_and_wait( pass1 ); - if( root ) { - root->my_body = temp_body; - root->my_incoming = NULL; - root->my_stuff_last = &body_; - task::spawn_root_and_wait( *root ); - } else { - body_.assign(temp_body->my_body); - temp_body->finish_construction( range_, NULL ); - temp_body->destroy(*temp_body); - } - } - } - }; - - template - task* start_scan::execute() { - typedef internal::finish_scan finish_pass1_type; - finish_pass1_type* p = my_parent_sum ? static_cast( parent() ) : NULL; - // Inspecting p->result.left_sum would ordinarily be a race condition. - // But we inspect it only if we are not a stolen task, in which case we - // know that task assigning to p->result.left_sum has completed. - bool treat_as_stolen = my_is_right_child && (is_stolen_task() || my_body!=p->my_result.my_left_sum); - if( treat_as_stolen ) { - // Invocation is for right child that has been really stolen or needs to be virtually stolen - p->my_right_zombie = my_body = new( allocate_root() ) final_sum_type(my_body->my_body); - my_is_final = false; - } - task* next_task = NULL; - if( (my_is_right_child && !treat_as_stolen) || !my_range.is_divisible() || my_partition.should_execute_range(*this) ) { - if( my_is_final ) - (my_body->my_body)( my_range, final_scan_tag() ); - else if( my_sum ) - (my_body->my_body)( my_range, pre_scan_tag() ); - if( my_sum ) - *my_sum = my_body; - __TBB_ASSERT( !*my_return_slot, NULL ); - } else { - sum_node_type* result; - if( my_parent_sum ) - result = new(allocate_additional_child_of(*my_parent_sum)) sum_node_type(my_range,/*my_left_is_final=*/my_is_final); - else - result = new(task::allocate_root()) sum_node_type(my_range,/*my_left_is_final=*/my_is_final); - finish_pass1_type& c = *new( allocate_continuation()) finish_pass1_type(*my_return_slot,my_sum,*result); - // Split off right child - start_scan& b = *new( c.allocate_child() ) start_scan( /*my_return_slot=*/result->my_right, *this, result ); - b.my_is_right_child = true; - // Left child is recycling of *this. Must recycle this before spawning b, - // otherwise b might complete and decrement c.ref_count() to zero, which - // would cause c.execute() to run prematurely. - recycle_as_child_of(c); - c.set_ref_count(2); - c.spawn(b); - my_sum = &result->my_left_sum; - my_return_slot = &result->my_left; - my_is_right_child = false; - next_task = this; - my_parent_sum = result; - __TBB_ASSERT( !*my_return_slot, NULL ); - } - return next_task; - } - - template - class lambda_scan_body : no_assign { - Value my_sum; - const Value& identity_element; - const Scan& my_scan; - const ReverseJoin& my_reverse_join; - public: - lambda_scan_body( const Value& identity, const Scan& scan, const ReverseJoin& rev_join) - : my_sum(identity) - , identity_element(identity) - , my_scan(scan) - , my_reverse_join(rev_join) {} - - lambda_scan_body( lambda_scan_body& b, split ) - : my_sum(b.identity_element) - , identity_element(b.identity_element) - , my_scan(b.my_scan) - , my_reverse_join(b.my_reverse_join) {} - - template - void operator()( const Range& r, Tag tag ) { - my_sum = my_scan(r, my_sum, tag); - } - - void reverse_join( lambda_scan_body& a ) { - my_sum = my_reverse_join(a.my_sum, my_sum); - } - - void assign( lambda_scan_body& b ) { - my_sum = b.my_sum; - } - - Value result() const { - return my_sum; - } - }; -} // namespace internal -//! @endcond - -// Requirements on Range concept are documented in blocked_range.h - -/** \page parallel_scan_body_req Requirements on parallel_scan body - Class \c Body implementing the concept of parallel_scan body must define: - - \code Body::Body( Body&, split ); \endcode Splitting constructor. - Split \c b so that \c this and \c b can accumulate separately - - \code Body::~Body(); \endcode Destructor - - \code void Body::operator()( const Range& r, pre_scan_tag ); \endcode - Preprocess iterations for range \c r - - \code void Body::operator()( const Range& r, final_scan_tag ); \endcode - Do final processing for iterations of range \c r - - \code void Body::reverse_join( Body& a ); \endcode - Merge preprocessing state of \c a into \c this, where \c a was - created earlier from \c b by b's splitting constructor -**/ - -/** \name parallel_scan - See also requirements on \ref range_req "Range" and \ref parallel_scan_body_req "parallel_scan Body". **/ -//@{ - -//! Parallel prefix with default partitioner -/** @ingroup algorithms **/ -template -void parallel_scan( const Range& range, Body& body ) { - internal::start_scan::run(range,body,__TBB_DEFAULT_PARTITIONER()); -} - -//! Parallel prefix with simple_partitioner -/** @ingroup algorithms **/ -template -void parallel_scan( const Range& range, Body& body, const simple_partitioner& partitioner ) { - internal::start_scan::run(range,body,partitioner); -} - -//! Parallel prefix with auto_partitioner -/** @ingroup algorithms **/ -template -void parallel_scan( const Range& range, Body& body, const auto_partitioner& partitioner ) { - internal::start_scan::run(range,body,partitioner); -} - -//! Parallel prefix with default partitioner -/** @ingroup algorithms **/ -template -Value parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join ) { - internal::lambda_scan_body body(identity, scan, reverse_join); - tbb::parallel_scan(range,body,__TBB_DEFAULT_PARTITIONER()); - return body.result(); -} - -//! Parallel prefix with simple_partitioner -/** @ingroup algorithms **/ -template -Value parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join, const simple_partitioner& partitioner ) { - internal::lambda_scan_body body(identity, scan, reverse_join); - tbb::parallel_scan(range,body,partitioner); - return body.result(); -} - -//! Parallel prefix with auto_partitioner -/** @ingroup algorithms **/ -template -Value parallel_scan( const Range& range, const Value& identity, const Scan& scan, const ReverseJoin& reverse_join, const auto_partitioner& partitioner ) { - internal::lambda_scan_body body(identity, scan, reverse_join); - tbb::parallel_scan(range,body,partitioner); - return body.result(); -} - -//@} - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_parallel_scan_H_include_area - -#endif /* __TBB_parallel_scan_H */ - +#include "../oneapi/tbb/parallel_scan.h" diff --git a/src/3rd party/tbb/parallel_sort.h b/src/3rd party/tbb/parallel_sort.h index b5f015ec29..b238e6caa4 100644 --- a/src/3rd party/tbb/parallel_sort.h +++ b/src/3rd party/tbb/parallel_sort.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,244 +14,4 @@ limitations under the License. */ -#ifndef __TBB_parallel_sort_H -#define __TBB_parallel_sort_H - -#define __TBB_parallel_sort_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "parallel_for.h" -#include "blocked_range.h" -#include "internal/_range_iterator.h" -#include -#include -#include -#if __TBB_TASK_GROUP_CONTEXT - #include "tbb_profiling.h" -#endif - -namespace tbb { - -namespace interface9 { -//! @cond INTERNAL -namespace internal { - -using tbb::internal::no_assign; - -//! Range used in quicksort to split elements into subranges based on a value. -/** The split operation selects a splitter and places all elements less than or equal - to the value in the first range and the remaining elements in the second range. - @ingroup algorithms */ -template -class quick_sort_range: private no_assign { - - inline size_t median_of_three(const RandomAccessIterator &array, size_t l, size_t m, size_t r) const { - return comp(array[l], array[m]) ? ( comp(array[m], array[r]) ? m : ( comp( array[l], array[r]) ? r : l ) ) - : ( comp(array[r], array[m]) ? m : ( comp( array[r], array[l] ) ? r : l ) ); - } - - inline size_t pseudo_median_of_nine( const RandomAccessIterator &array, const quick_sort_range &range ) const { - size_t offset = range.size/8u; - return median_of_three(array, - median_of_three(array, 0, offset, offset*2), - median_of_three(array, offset*3, offset*4, offset*5), - median_of_three(array, offset*6, offset*7, range.size - 1) ); - - } - - size_t split_range( quick_sort_range& range ) { - using std::iter_swap; - RandomAccessIterator array = range.begin; - RandomAccessIterator key0 = range.begin; - size_t m = pseudo_median_of_nine(array, range); - if (m) iter_swap ( array, array+m ); - - size_t i=0; - size_t j=range.size; - // Partition interval [i+1,j-1] with key *key0. - for(;;) { - __TBB_ASSERT( i=grainsize;} - - quick_sort_range( quick_sort_range& range, split ) - : comp(range.comp) - , size(split_range(range)) - // +1 accounts for the pivot element, which is at its correct place - // already and, therefore, is not included into subranges. - , begin(range.begin+range.size+1) {} -}; - -#if __TBB_TASK_GROUP_CONTEXT -//! Body class used to test if elements in a range are presorted -/** @ingroup algorithms */ -template -class quick_sort_pretest_body : no_assign { - const Compare ∁ - -public: - quick_sort_pretest_body(const Compare &_comp) : comp(_comp) {} - - void operator()( const blocked_range& range ) const { - task &my_task = task::self(); - RandomAccessIterator my_end = range.end(); - - int i = 0; - for (RandomAccessIterator k = range.begin(); k != my_end; ++k, ++i) { - if ( i%64 == 0 && my_task.is_cancelled() ) break; - - // The k-1 is never out-of-range because the first chunk starts at begin+serial_cutoff+1 - if ( comp( *(k), *(k-1) ) ) { - my_task.cancel_group_execution(); - break; - } - } - } - -}; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -//! Body class used to sort elements in a range that is smaller than the grainsize. -/** @ingroup algorithms */ -template -struct quick_sort_body { - void operator()( const quick_sort_range& range ) const { - //SerialQuickSort( range.begin, range.size, range.comp ); - std::sort( range.begin, range.begin + range.size, range.comp ); - } -}; - -//! Wrapper method to initiate the sort by calling parallel_for. -/** @ingroup algorithms */ -template -void parallel_quick_sort( RandomAccessIterator begin, RandomAccessIterator end, const Compare& comp ) { -#if __TBB_TASK_GROUP_CONTEXT - task_group_context my_context(PARALLEL_SORT); - const int serial_cutoff = 9; - - __TBB_ASSERT( begin + serial_cutoff < end, "min_parallel_size is smaller than serial cutoff?" ); - RandomAccessIterator k = begin; - for ( ; k != begin + serial_cutoff; ++k ) { - if ( comp( *(k+1), *k ) ) { - goto do_parallel_quick_sort; - } - } - - parallel_for( blocked_range(k+1, end), - quick_sort_pretest_body(comp), - auto_partitioner(), - my_context); - - if (my_context.is_group_execution_cancelled()) -do_parallel_quick_sort: -#endif /* __TBB_TASK_GROUP_CONTEXT */ - parallel_for( quick_sort_range(begin, end-begin, comp ), - quick_sort_body(), - auto_partitioner() ); -} - -} // namespace internal -//! @endcond -} // namespace interfaceX - -/** \page parallel_sort_iter_req Requirements on iterators for parallel_sort - Requirements on the iterator type \c It and its value type \c T for \c parallel_sort: - - - \code void iter_swap( It a, It b ) \endcode Swaps the values of the elements the given - iterators \c a and \c b are pointing to. \c It should be a random access iterator. - - - \code bool Compare::operator()( const T& x, const T& y ) \endcode True if x comes before y; -**/ - -/** \name parallel_sort - See also requirements on \ref parallel_sort_iter_req "iterators for parallel_sort". **/ -//@{ - -//! Sorts the data in [begin,end) using the given comparator -/** The compare function object is used for all comparisons between elements during sorting. - The compare object must define a bool operator() function. - @ingroup algorithms **/ -template -void parallel_sort( RandomAccessIterator begin, RandomAccessIterator end, const Compare& comp) { - const int min_parallel_size = 500; - if( end > begin ) { - if (end - begin < min_parallel_size) { - std::sort(begin, end, comp); - } else { - interface9::internal::parallel_quick_sort(begin, end, comp); - } - } -} - -//! Sorts the data in [begin,end) with a default comparator \c std::less -/** @ingroup algorithms **/ -template -inline void parallel_sort( RandomAccessIterator begin, RandomAccessIterator end ) { - parallel_sort( begin, end, std::less< typename std::iterator_traits::value_type >() ); -} - -//! Sorts the data in rng using the given comparator -/** @ingroup algorithms **/ -template -void parallel_sort(Range& rng, const Compare& comp) { - parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng), comp); -} - -//! Sorts the data in rng with a default comparator \c std::less -/** @ingroup algorithms **/ -template -void parallel_sort(Range& rng) { - parallel_sort(tbb::internal::first(rng), tbb::internal::last(rng)); -} - -//! Sorts the data in the range \c [begin,end) with a default comparator \c std::less -/** @ingroup algorithms **/ -template -inline void parallel_sort( T * begin, T * end ) { - parallel_sort( begin, end, std::less< T >() ); -} -//@} - - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_parallel_sort_H_include_area - -#endif - +#include "../oneapi/tbb/parallel_sort.h" diff --git a/src/3rd party/tbb/parallel_while.h b/src/3rd party/tbb/parallel_while.h deleted file mode 100644 index b32c7817a6..0000000000 --- a/src/3rd party/tbb/parallel_while.h +++ /dev/null @@ -1,188 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_parallel_while -#define __TBB_parallel_while - -#define __TBB_parallel_while_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "task.h" -#include - -namespace tbb { - -template -class parallel_while; - -//! @cond INTERNAL -namespace internal { - - template class while_task; - - //! For internal use only. - /** Executes one iteration of a while. - @ingroup algorithms */ - template - class while_iteration_task: public task { - const Body& my_body; - typename Body::argument_type my_value; - task* execute() __TBB_override { - my_body(my_value); - return NULL; - } - while_iteration_task( const typename Body::argument_type& value, const Body& body ) : - my_body(body), my_value(value) - {} - template friend class while_group_task; - friend class tbb::parallel_while; - }; - - //! For internal use only - /** Unpacks a block of iterations. - @ingroup algorithms */ - template - class while_group_task: public task { - static const size_t max_arg_size = 4; - const Body& my_body; - size_t size; - typename Body::argument_type my_arg[max_arg_size]; - while_group_task( const Body& body ) : my_body(body), size(0) {} - task* execute() __TBB_override { - typedef while_iteration_task iteration_type; - __TBB_ASSERT( size>0, NULL ); - task_list list; - task* t; - size_t k=0; - for(;;) { - t = new( allocate_child() ) iteration_type(my_arg[k],my_body); - if( ++k==size ) break; - list.push_back(*t); - } - set_ref_count(int(k+1)); - spawn(list); - spawn_and_wait_for_all(*t); - return NULL; - } - template friend class while_task; - }; - - //! For internal use only. - /** Gets block of iterations from a stream and packages them into a while_group_task. - @ingroup algorithms */ - template - class while_task: public task { - Stream& my_stream; - const Body& my_body; - empty_task& my_barrier; - task* execute() __TBB_override { - typedef while_group_task block_type; - block_type& t = *new( allocate_additional_child_of(my_barrier) ) block_type(my_body); - size_t k=0; - while( my_stream.pop_if_present(t.my_arg[k]) ) { - if( ++k==block_type::max_arg_size ) { - // There might be more iterations. - recycle_to_reexecute(); - break; - } - } - if( k==0 ) { - destroy(t); - return NULL; - } else { - t.size = k; - return &t; - } - } - while_task( Stream& stream, const Body& body, empty_task& barrier ) : - my_stream(stream), - my_body(body), - my_barrier(barrier) - {} - friend class tbb::parallel_while; - }; - -} // namespace internal -//! @endcond - -//! Parallel iteration over a stream, with optional addition of more work. -/** The Body b has the requirement: \n - "b(v)" \n - "b.argument_type" \n - where v is an argument_type - @ingroup algorithms */ -template -class parallel_while: internal::no_copy { -public: - //! Construct empty non-running parallel while. - parallel_while() : my_body(NULL), my_barrier(NULL) {} - - //! Destructor cleans up data members before returning. - ~parallel_while() { - if( my_barrier ) { - my_barrier->destroy(*my_barrier); - my_barrier = NULL; - } - } - - //! Type of items - typedef typename Body::argument_type value_type; - - //! Apply body.apply to each item in the stream. - /** A Stream s has the requirements \n - "S::value_type" \n - "s.pop_if_present(value) is convertible to bool */ - template - void run( Stream& stream, const Body& body ); - - //! Add a work item while running. - /** Should be executed only by body.apply or a thread spawned therefrom. */ - void add( const value_type& item ); - -private: - const Body* my_body; - empty_task* my_barrier; -}; - -template -template -void parallel_while::run( Stream& stream, const Body& body ) { - using namespace internal; - empty_task& barrier = *new( task::allocate_root() ) empty_task(); - my_body = &body; - my_barrier = &barrier; - my_barrier->set_ref_count(2); - while_task& w = *new( my_barrier->allocate_child() ) while_task( stream, body, barrier ); - my_barrier->spawn_and_wait_for_all(w); - my_barrier->destroy(*my_barrier); - my_barrier = NULL; - my_body = NULL; -} - -template -void parallel_while::add( const value_type& item ) { - __TBB_ASSERT(my_barrier,"attempt to add to parallel_while that is not running"); - typedef internal::while_iteration_task iteration_type; - iteration_type& i = *new( task::allocate_additional_child_of(*my_barrier) ) iteration_type(item,*my_body); - task::self().spawn( i ); -} - -} // namespace - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_parallel_while_H_include_area - -#endif /* __TBB_parallel_while */ diff --git a/src/3rd party/tbb/partitioner.h b/src/3rd party/tbb/partitioner.h index b17c26341c..b959e35a2f 100644 --- a/src/3rd party/tbb/partitioner.h +++ b/src/3rd party/tbb/partitioner.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,668 +14,4 @@ limitations under the License. */ -#ifndef __TBB_partitioner_H -#define __TBB_partitioner_H - -#define __TBB_partitioner_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#ifndef __TBB_INITIAL_CHUNKS -// initial task divisions per thread -#define __TBB_INITIAL_CHUNKS 2 -#endif -#ifndef __TBB_RANGE_POOL_CAPACITY -// maximum number of elements in range pool -#define __TBB_RANGE_POOL_CAPACITY 8 -#endif -#ifndef __TBB_INIT_DEPTH -// initial value for depth of range pool -#define __TBB_INIT_DEPTH 5 -#endif -#ifndef __TBB_DEMAND_DEPTH_ADD -// when imbalance is found range splits this value times more -#define __TBB_DEMAND_DEPTH_ADD 1 -#endif -#ifndef __TBB_STATIC_THRESHOLD -// necessary number of clocks for the work to be distributed among all tasks -#define __TBB_STATIC_THRESHOLD 40000 -#endif -#if __TBB_DEFINE_MIC -#define __TBB_NONUNIFORM_TASK_CREATION 1 -#ifdef __TBB_time_stamp -#define __TBB_USE_MACHINE_TIME_STAMPS 1 -#define __TBB_task_duration() __TBB_STATIC_THRESHOLD -#endif // __TBB_machine_time_stamp -#endif // __TBB_DEFINE_MIC - -#include "task.h" -#include "task_arena.h" -#include "aligned_space.h" -#include "atomic.h" -#include "internal/_template_helpers.h" - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - // Workaround for overzealous compiler warnings - #pragma warning (push) - #pragma warning (disable: 4244) -#endif - -namespace tbb { - -class auto_partitioner; -class simple_partitioner; -class static_partitioner; -class affinity_partitioner; - -namespace interface9 { - namespace internal { - class affinity_partition_type; - } -} - -namespace internal { //< @cond INTERNAL -size_t __TBB_EXPORTED_FUNC get_initial_auto_partitioner_divisor(); - -//! Defines entry point for affinity partitioner into tbb run-time library. -class affinity_partitioner_base_v3: no_copy { - friend class tbb::affinity_partitioner; - friend class tbb::interface9::internal::affinity_partition_type; - //! Array that remembers affinities of tree positions to affinity_id. - /** NULL if my_size==0. */ - affinity_id* my_array; - //! Number of elements in my_array. - size_t my_size; - //! Zeros the fields. - affinity_partitioner_base_v3() : my_array(NULL), my_size(0) {} - //! Deallocates my_array. - ~affinity_partitioner_base_v3() {resize(0);} - //! Resize my_array. - /** Retains values if resulting size is the same. */ - void __TBB_EXPORTED_METHOD resize( unsigned factor ); -}; - -//! Provides backward-compatible methods for partition objects without affinity. -class partition_type_base { -public: - void set_affinity( task & ) {} - void note_affinity( task::affinity_id ) {} - task* continue_after_execute_range() {return NULL;} - bool decide_whether_to_delay() {return false;} - void spawn_or_delay( bool, task& b ) { - task::spawn(b); - } -}; - -template class start_scan; - -} //< namespace internal @endcond - -namespace serial { -namespace interface9 { -template class start_for; -} -} - -namespace interface9 { -//! @cond INTERNAL -namespace internal { -using namespace tbb::internal; -template class start_for; -template class start_reduce; -template class start_deterministic_reduce; - -//! Join task node that contains shared flag for stealing feedback -class flag_task: public task { -public: - tbb::atomic my_child_stolen; - flag_task() { my_child_stolen = false; } - task* execute() __TBB_override { return NULL; } - static void mark_task_stolen(task &t) { - tbb::atomic &flag = static_cast(t.parent())->my_child_stolen; -#if TBB_USE_THREADING_TOOLS - // Threading tools respect lock prefix but report false-positive data-race via plain store - flag.fetch_and_store(true); -#else - flag = true; -#endif //TBB_USE_THREADING_TOOLS - } - static bool is_peer_stolen(task &t) { - return static_cast(t.parent())->my_child_stolen; - } -}; - -//! Depth is a relative depth of recursive division inside a range pool. Relative depth allows -//! infinite absolute depth of the recursion for heavily unbalanced workloads with range represented -//! by a number that cannot fit into machine word. -typedef unsigned char depth_t; - -//! Range pool stores ranges of type T in a circular buffer with MaxCapacity -template -class range_vector { - depth_t my_head; - depth_t my_tail; - depth_t my_size; - depth_t my_depth[MaxCapacity]; // relative depths of stored ranges - tbb::aligned_space my_pool; - -public: - //! initialize via first range in pool - range_vector(const T& elem) : my_head(0), my_tail(0), my_size(1) { - my_depth[0] = 0; - new( static_cast(my_pool.begin()) ) T(elem);//TODO: std::move? - } - ~range_vector() { - while( !empty() ) pop_back(); - } - bool empty() const { return my_size == 0; } - depth_t size() const { return my_size; } - //! Populates range pool via ranges up to max depth or while divisible - //! max_depth starts from 0, e.g. value 2 makes 3 ranges in the pool up to two 1/4 pieces - void split_to_fill(depth_t max_depth) { - while( my_size < MaxCapacity && is_divisible(max_depth) ) { - depth_t prev = my_head; - my_head = (my_head + 1) % MaxCapacity; - new(my_pool.begin()+my_head) T(my_pool.begin()[prev]); // copy TODO: std::move? - my_pool.begin()[prev].~T(); // instead of assignment - new(my_pool.begin()+prev) T(my_pool.begin()[my_head], split()); // do 'inverse' split - my_depth[my_head] = ++my_depth[prev]; - my_size++; - } - } - void pop_back() { - __TBB_ASSERT(my_size > 0, "range_vector::pop_back() with empty size"); - my_pool.begin()[my_head].~T(); - my_size--; - my_head = (my_head + MaxCapacity - 1) % MaxCapacity; - } - void pop_front() { - __TBB_ASSERT(my_size > 0, "range_vector::pop_front() with empty size"); - my_pool.begin()[my_tail].~T(); - my_size--; - my_tail = (my_tail + 1) % MaxCapacity; - } - T& back() { - __TBB_ASSERT(my_size > 0, "range_vector::back() with empty size"); - return my_pool.begin()[my_head]; - } - T& front() { - __TBB_ASSERT(my_size > 0, "range_vector::front() with empty size"); - return my_pool.begin()[my_tail]; - } - //! similarly to front(), returns depth of the first range in the pool - depth_t front_depth() { - __TBB_ASSERT(my_size > 0, "range_vector::front_depth() with empty size"); - return my_depth[my_tail]; - } - depth_t back_depth() { - __TBB_ASSERT(my_size > 0, "range_vector::back_depth() with empty size"); - return my_depth[my_head]; - } - bool is_divisible(depth_t max_depth) { - return back_depth() < max_depth && back().is_divisible(); - } -}; - -//! Provides default methods for partition objects and common algorithm blocks. -template -struct partition_type_base { - typedef split split_type; - // decision makers - void set_affinity( task & ) {} - void note_affinity( task::affinity_id ) {} - bool check_being_stolen(task &) { return false; } // part of old should_execute_range() - bool check_for_demand(task &) { return false; } - bool is_divisible() { return true; } // part of old should_execute_range() - depth_t max_depth() { return 0; } - void align_depth(depth_t) { } - template split_type get_split() { return split(); } - Partition& self() { return *static_cast(this); } // CRTP helper - - template - void work_balance(StartType &start, Range &range) { - start.run_body( range ); // simple partitioner goes always here - } - - template - void execute(StartType &start, Range &range) { - // The algorithm in a few words ([]-denotes calls to decision methods of partitioner): - // [If this task is stolen, adjust depth and divisions if necessary, set flag]. - // If range is divisible { - // Spread the work while [initial divisions left]; - // Create trap task [if necessary]; - // } - // If not divisible or [max depth is reached], execute, else do the range pool part - if ( range.is_divisible() ) { - if ( self().is_divisible() ) { - do { // split until is divisible - typename Partition::split_type split_obj = self().template get_split(); - start.offer_work( split_obj ); - } while ( range.is_divisible() && self().is_divisible() ); - } - } - self().work_balance(start, range); - } -}; - -//! Provides default splitting strategy for partition objects. -template -struct adaptive_mode : partition_type_base { - typedef Partition my_partition; - size_t my_divisor; - // For affinity_partitioner, my_divisor indicates the number of affinity array indices the task reserves. - // A task which has only one index must produce the right split without reserved index in order to avoid - // it to be overwritten in note_affinity() of the created (right) task. - // I.e. a task created deeper than the affinity array can remember must not save its affinity (LIFO order) - static const unsigned factor = 1; - adaptive_mode() : my_divisor(tbb::internal::get_initial_auto_partitioner_divisor() / 4 * my_partition::factor) {} - adaptive_mode(adaptive_mode &src, split) : my_divisor(do_split(src, split())) {} - /*! Override do_split methods in order to specify splitting strategy */ - size_t do_split(adaptive_mode &src, split) { - return src.my_divisor /= 2u; - } -}; - -//! A helper class to create a proportional_split object for a given type of Range. -/** If the Range has static boolean constant 'is_splittable_in_proportion' set to 'true', - the created object splits a provided value in an implemenation-defined proportion; - otherwise it represents equal-size split. */ -// TODO: check if this helper can be a nested class of proportional_mode. -template -struct proportion_helper { - static proportional_split get_split(size_t) { return proportional_split(1,1); } -}; -template -struct proportion_helper::type> { - static proportional_split get_split(size_t n) { -#if __TBB_NONUNIFORM_TASK_CREATION - size_t right = (n + 2) / 3; -#else - size_t right = n / 2; -#endif - size_t left = n - right; - return proportional_split(left, right); - } -}; - -//! Provides proportional splitting strategy for partition objects -template -struct proportional_mode : adaptive_mode { - typedef Partition my_partition; - using partition_type_base::self; // CRTP helper to get access to derived classes - - proportional_mode() : adaptive_mode() {} - proportional_mode(proportional_mode &src, split) : adaptive_mode(src, split()) {} - proportional_mode(proportional_mode &src, const proportional_split& split_obj) { self().my_divisor = do_split(src, split_obj); } - size_t do_split(proportional_mode &src, const proportional_split& split_obj) { -#if __TBB_ENABLE_RANGE_FEEDBACK - size_t portion = size_t(float(src.my_divisor) * float(split_obj.right()) - / float(split_obj.left() + split_obj.right()) + 0.5f); -#else - size_t portion = split_obj.right() * my_partition::factor; -#endif - portion = (portion + my_partition::factor/2) & (0ul - my_partition::factor); -#if __TBB_ENABLE_RANGE_FEEDBACK - /** Corner case handling */ - if (!portion) - portion = my_partition::factor; - else if (portion == src.my_divisor) - portion = src.my_divisor - my_partition::factor; -#endif - src.my_divisor -= portion; - return portion; - } - bool is_divisible() { // part of old should_execute_range() - return self().my_divisor > my_partition::factor; - } - template - proportional_split get_split() { - // Create a proportion for the number of threads expected to handle "this" subrange - return proportion_helper::get_split( self().my_divisor / my_partition::factor ); - } -}; - -static size_t get_initial_partition_head() { - int current_index = tbb::this_task_arena::current_thread_index(); - if (current_index == tbb::task_arena::not_initialized) - current_index = 0; - return size_t(current_index); -} - -//! Provides default linear indexing of partitioner's sequence -template -struct linear_affinity_mode : proportional_mode { - size_t my_head; - size_t my_max_affinity; - using proportional_mode::self; - linear_affinity_mode() : proportional_mode(), my_head(get_initial_partition_head()), - my_max_affinity(self().my_divisor) {} - linear_affinity_mode(linear_affinity_mode &src, split) : proportional_mode(src, split()) - , my_head((src.my_head + src.my_divisor) % src.my_max_affinity), my_max_affinity(src.my_max_affinity) {} - linear_affinity_mode(linear_affinity_mode &src, const proportional_split& split_obj) : proportional_mode(src, split_obj) - , my_head((src.my_head + src.my_divisor) % src.my_max_affinity), my_max_affinity(src.my_max_affinity) {} - void set_affinity( task &t ) { - if( self().my_divisor ) - t.set_affinity( affinity_id(my_head) + 1 ); - } -}; - -/*! Determine work-balance phase implementing splitting & stealing actions */ -template -struct dynamic_grainsize_mode : Mode { - using Mode::self; -#ifdef __TBB_USE_MACHINE_TIME_STAMPS - tbb::internal::machine_tsc_t my_dst_tsc; -#endif - enum { - begin = 0, - run, - pass - } my_delay; - depth_t my_max_depth; - static const unsigned range_pool_size = __TBB_RANGE_POOL_CAPACITY; - dynamic_grainsize_mode(): Mode() -#ifdef __TBB_USE_MACHINE_TIME_STAMPS - , my_dst_tsc(0) -#endif - , my_delay(begin) - , my_max_depth(__TBB_INIT_DEPTH) {} - dynamic_grainsize_mode(dynamic_grainsize_mode& p, split) - : Mode(p, split()) -#ifdef __TBB_USE_MACHINE_TIME_STAMPS - , my_dst_tsc(0) -#endif - , my_delay(pass) - , my_max_depth(p.my_max_depth) {} - dynamic_grainsize_mode(dynamic_grainsize_mode& p, const proportional_split& split_obj) - : Mode(p, split_obj) -#ifdef __TBB_USE_MACHINE_TIME_STAMPS - , my_dst_tsc(0) -#endif - , my_delay(begin) - , my_max_depth(p.my_max_depth) {} - bool check_being_stolen(task &t) { // part of old should_execute_range() - if( !(self().my_divisor / Mode::my_partition::factor) ) { // if not from the top P tasks of binary tree - self().my_divisor = 1; // TODO: replace by on-stack flag (partition_state's member)? - if( t.is_stolen_task() && t.parent()->ref_count() >= 2 ) { // runs concurrently with the left task -#if __TBB_USE_OPTIONAL_RTTI - // RTTI is available, check whether the cast is valid - __TBB_ASSERT(dynamic_cast(t.parent()), 0); - // correctness of the cast relies on avoiding the root task for which: - // - initial value of my_divisor != 0 (protected by separate assertion) - // - is_stolen_task() always returns false for the root task. -#endif - flag_task::mark_task_stolen(t); - if( !my_max_depth ) my_max_depth++; - my_max_depth += __TBB_DEMAND_DEPTH_ADD; - return true; - } - } - return false; - } - depth_t max_depth() { return my_max_depth; } - void align_depth(depth_t base) { - __TBB_ASSERT(base <= my_max_depth, 0); - my_max_depth -= base; - } - template - void work_balance(StartType &start, Range &range) { - if( !range.is_divisible() || !self().max_depth() ) { - start.run_body( range ); // simple partitioner goes always here - } - else { // do range pool - internal::range_vector range_pool(range); - do { - range_pool.split_to_fill(self().max_depth()); // fill range pool - if( self().check_for_demand( start ) ) { - if( range_pool.size() > 1 ) { - start.offer_work( range_pool.front(), range_pool.front_depth() ); - range_pool.pop_front(); - continue; - } - if( range_pool.is_divisible(self().max_depth()) ) // was not enough depth to fork a task - continue; // note: next split_to_fill() should split range at least once - } - start.run_body( range_pool.back() ); - range_pool.pop_back(); - } while( !range_pool.empty() && !start.is_cancelled() ); - } - } - bool check_for_demand( task &t ) { - if( pass == my_delay ) { - if( self().my_divisor > 1 ) // produce affinitized tasks while they have slot in array - return true; // do not do my_max_depth++ here, but be sure range_pool is splittable once more - else if( self().my_divisor && my_max_depth ) { // make balancing task - self().my_divisor = 0; // once for each task; depth will be decreased in align_depth() - return true; - } - else if( flag_task::is_peer_stolen(t) ) { - my_max_depth += __TBB_DEMAND_DEPTH_ADD; - return true; - } - } else if( begin == my_delay ) { -#ifndef __TBB_USE_MACHINE_TIME_STAMPS - my_delay = pass; -#else - my_dst_tsc = __TBB_time_stamp() + __TBB_task_duration(); - my_delay = run; - } else if( run == my_delay ) { - if( __TBB_time_stamp() < my_dst_tsc ) { - __TBB_ASSERT(my_max_depth > 0, NULL); - my_max_depth--; // increase granularity since tasks seem having too small work - return false; - } - my_delay = pass; - return true; -#endif // __TBB_USE_MACHINE_TIME_STAMPS - } - return false; - } -}; - -class auto_partition_type: public dynamic_grainsize_mode > { -public: - auto_partition_type( const auto_partitioner& ) - : dynamic_grainsize_mode >() { - my_divisor *= __TBB_INITIAL_CHUNKS; - } - auto_partition_type( auto_partition_type& src, split) - : dynamic_grainsize_mode >(src, split()) {} - bool is_divisible() { // part of old should_execute_range() - if( my_divisor > 1 ) return true; - if( my_divisor && my_max_depth ) { // can split the task. TODO: on-stack flag instead - // keep same fragmentation while splitting for the local task pool - my_max_depth--; - my_divisor = 0; // decrease max_depth once per task - return true; - } else return false; - } - bool check_for_demand(task &t) { - if( flag_task::is_peer_stolen(t) ) { - my_max_depth += __TBB_DEMAND_DEPTH_ADD; - return true; - } else return false; - } -}; - -class simple_partition_type: public partition_type_base { -public: - simple_partition_type( const simple_partitioner& ) {} - simple_partition_type( const simple_partition_type&, split ) {} - //! simplified algorithm - template - void execute(StartType &start, Range &range) { - split_type split_obj = split(); // start.offer_work accepts split_type as reference - while( range.is_divisible() ) - start.offer_work( split_obj ); - start.run_body( range ); - } -}; - -class static_partition_type : public linear_affinity_mode { -public: - typedef proportional_split split_type; - static_partition_type( const static_partitioner& ) - : linear_affinity_mode() {} - static_partition_type( static_partition_type& p, split ) - : linear_affinity_mode(p, split()) {} - static_partition_type( static_partition_type& p, const proportional_split& split_obj ) - : linear_affinity_mode(p, split_obj) {} -}; - -class affinity_partition_type : public dynamic_grainsize_mode > { - static const unsigned factor_power = 4; // TODO: get a unified formula based on number of computing units - tbb::internal::affinity_id* my_array; -public: - static const unsigned factor = 1 << factor_power; // number of slots in affinity array per task - typedef proportional_split split_type; - affinity_partition_type( tbb::internal::affinity_partitioner_base_v3& ap ) - : dynamic_grainsize_mode >() { - __TBB_ASSERT( (factor&(factor-1))==0, "factor must be power of two" ); - ap.resize(factor); - my_array = ap.my_array; - my_max_depth = factor_power + 1; - __TBB_ASSERT( my_max_depth < __TBB_RANGE_POOL_CAPACITY, 0 ); - } - affinity_partition_type(affinity_partition_type& p, split) - : dynamic_grainsize_mode >(p, split()) - , my_array(p.my_array) {} - affinity_partition_type(affinity_partition_type& p, const proportional_split& split_obj) - : dynamic_grainsize_mode >(p, split_obj) - , my_array(p.my_array) {} - void set_affinity( task &t ) { - if( my_divisor ) { - if( !my_array[my_head] ) - // TODO: consider new ideas with my_array for both affinity and static partitioner's, then code reuse - t.set_affinity( affinity_id(my_head / factor + 1) ); - else - t.set_affinity( my_array[my_head] ); - } - } - void note_affinity( task::affinity_id id ) { - if( my_divisor ) - my_array[my_head] = id; - } -}; - -//! Backward-compatible partition for auto and affinity partition objects. -class old_auto_partition_type: public tbb::internal::partition_type_base { - size_t num_chunks; - static const size_t VICTIM_CHUNKS = 4; -public: - bool should_execute_range(const task &t) { - if( num_chunks friend class serial::interface9::start_for; - template friend class interface9::internal::start_for; - template friend class interface9::internal::start_reduce; - template friend class interface9::internal::start_deterministic_reduce; - template friend class internal::start_scan; - // backward compatibility - class partition_type: public internal::partition_type_base { - public: - bool should_execute_range(const task& ) {return false;} - partition_type( const simple_partitioner& ) {} - partition_type( const partition_type&, split ) {} - }; - // new implementation just extends existing interface - typedef interface9::internal::simple_partition_type task_partition_type; - - // TODO: consider to make split_type public - typedef interface9::internal::simple_partition_type::split_type split_type; -}; - -//! An auto partitioner -/** The range is initial divided into several large chunks. - Chunks are further subdivided into smaller pieces if demand detected and they are divisible. - @ingroup algorithms */ -class auto_partitioner { -public: - auto_partitioner() {} - -private: - template friend class serial::interface9::start_for; - template friend class interface9::internal::start_for; - template friend class interface9::internal::start_reduce; - template friend class internal::start_scan; - // backward compatibility - typedef interface9::internal::old_auto_partition_type partition_type; - // new implementation just extends existing interface - typedef interface9::internal::auto_partition_type task_partition_type; - - // TODO: consider to make split_type public - typedef interface9::internal::auto_partition_type::split_type split_type; -}; - -//! A static partitioner -class static_partitioner { -public: - static_partitioner() {} -private: - template friend class serial::interface9::start_for; - template friend class interface9::internal::start_for; - template friend class interface9::internal::start_reduce; - template friend class interface9::internal::start_deterministic_reduce; - template friend class internal::start_scan; - // backward compatibility - typedef interface9::internal::old_auto_partition_type partition_type; - // new implementation just extends existing interface - typedef interface9::internal::static_partition_type task_partition_type; - - // TODO: consider to make split_type public - typedef interface9::internal::static_partition_type::split_type split_type; -}; - -//! An affinity partitioner -class affinity_partitioner: internal::affinity_partitioner_base_v3 { -public: - affinity_partitioner() {} - -private: - template friend class serial::interface9::start_for; - template friend class interface9::internal::start_for; - template friend class interface9::internal::start_reduce; - template friend class internal::start_scan; - // backward compatibility - for parallel_scan only - typedef interface9::internal::old_auto_partition_type partition_type; - // new implementation just extends existing interface - typedef interface9::internal::affinity_partition_type task_partition_type; - - // TODO: consider to make split_type public - typedef interface9::internal::affinity_partition_type::split_type split_type; -}; - -} // namespace tbb - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warning 4244 is back -#undef __TBB_INITIAL_CHUNKS -#undef __TBB_RANGE_POOL_CAPACITY -#undef __TBB_INIT_DEPTH - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_partitioner_H_include_area - -#endif /* __TBB_partitioner_H */ +#include "../oneapi/tbb/partitioner.h" diff --git a/src/3rd party/tbb/pipeline.h b/src/3rd party/tbb/pipeline.h deleted file mode 100644 index fad817b11f..0000000000 --- a/src/3rd party/tbb/pipeline.h +++ /dev/null @@ -1,682 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_pipeline_H -#define __TBB_pipeline_H - -#define __TBB_pipeline_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "atomic.h" -#include "task.h" -#include "tbb_allocator.h" -#include - -#if __TBB_CPP11_TYPE_PROPERTIES_PRESENT -#include -#endif - -namespace tbb { - -class pipeline; -class filter; - -//! @cond INTERNAL -namespace internal { - -// The argument for PIPELINE_VERSION should be an integer between 2 and 9 -#define __TBB_PIPELINE_VERSION(x) ((unsigned char)(x-2)<<1) - -typedef unsigned long Token; -typedef long tokendiff_t; -class stage_task; -class input_buffer; -class pipeline_root_task; -class pipeline_cleaner; - -} // namespace internal - -namespace interface6 { - template class filter_t; - - namespace internal { - class pipeline_proxy; - } -} - -//! @endcond - -//! A stage in a pipeline. -/** @ingroup algorithms */ -class filter: internal::no_copy { -private: - //! Value used to mark "not in pipeline" - static filter* not_in_pipeline() { return reinterpret_cast(intptr_t(-1)); } -protected: - //! The lowest bit 0 is for parallel vs. serial - static const unsigned char filter_is_serial = 0x1; - - //! 4th bit distinguishes ordered vs unordered filters. - /** The bit was not set for parallel filters in TBB 2.1 and earlier, - but is_ordered() function always treats parallel filters as out of order. */ - static const unsigned char filter_is_out_of_order = 0x1<<4; - - //! 5th bit distinguishes thread-bound and regular filters. - static const unsigned char filter_is_bound = 0x1<<5; - - //! 6th bit marks input filters emitting small objects - static const unsigned char filter_may_emit_null = 0x1<<6; - - //! 7th bit defines exception propagation mode expected by the application. - static const unsigned char exact_exception_propagation = -#if TBB_USE_CAPTURED_EXCEPTION - 0x0; -#else - 0x1<<7; -#endif /* TBB_USE_CAPTURED_EXCEPTION */ - - static const unsigned char current_version = __TBB_PIPELINE_VERSION(5); - static const unsigned char version_mask = 0x7<<1; // bits 1-3 are for version -public: - enum mode { - //! processes multiple items in parallel and in no particular order - parallel = current_version | filter_is_out_of_order, - //! processes items one at a time; all such filters process items in the same order - serial_in_order = current_version | filter_is_serial, - //! processes items one at a time and in no particular order - serial_out_of_order = current_version | filter_is_serial | filter_is_out_of_order, - //! @deprecated use serial_in_order instead - serial = serial_in_order - }; -protected: - explicit filter( bool is_serial_ ) : - next_filter_in_pipeline(not_in_pipeline()), - my_input_buffer(NULL), - my_filter_mode(static_cast((is_serial_ ? serial : parallel) | exact_exception_propagation)), - prev_filter_in_pipeline(not_in_pipeline()), - my_pipeline(NULL), - next_segment(NULL) - {} - - explicit filter( mode filter_mode ) : - next_filter_in_pipeline(not_in_pipeline()), - my_input_buffer(NULL), - my_filter_mode(static_cast(filter_mode | exact_exception_propagation)), - prev_filter_in_pipeline(not_in_pipeline()), - my_pipeline(NULL), - next_segment(NULL) - {} - - // signal end-of-input for concrete_filters - void __TBB_EXPORTED_METHOD set_end_of_input(); - -public: - //! True if filter is serial. - bool is_serial() const { - return bool( my_filter_mode & filter_is_serial ); - } - - //! True if filter must receive stream in order. - bool is_ordered() const { - return (my_filter_mode & (filter_is_out_of_order|filter_is_serial))==filter_is_serial; - } - - //! True if filter is thread-bound. - bool is_bound() const { - return ( my_filter_mode & filter_is_bound )==filter_is_bound; - } - - //! true if an input filter can emit null - bool object_may_be_null() { - return ( my_filter_mode & filter_may_emit_null ) == filter_may_emit_null; - } - - //! Operate on an item from the input stream, and return item for output stream. - /** Returns NULL if filter is a sink. */ - virtual void* operator()( void* item ) = 0; - - //! Destroy filter. - /** If the filter was added to a pipeline, the pipeline must be destroyed first. */ - virtual __TBB_EXPORTED_METHOD ~filter(); - -#if __TBB_TASK_GROUP_CONTEXT - //! Destroys item if pipeline was cancelled. - /** Required to prevent memory leaks. - Note it can be called concurrently even for serial filters.*/ - virtual void finalize( void* /*item*/ ) {} -#endif - -private: - //! Pointer to next filter in the pipeline. - filter* next_filter_in_pipeline; - - //! has the filter not yet processed all the tokens it will ever see? - // (pipeline has not yet reached end_of_input or this filter has not yet - // seen the last token produced by input_filter) - bool has_more_work(); - - //! Buffer for incoming tokens, or NULL if not required. - /** The buffer is required if the filter is serial or follows a thread-bound one. */ - internal::input_buffer* my_input_buffer; - - friend class internal::stage_task; - friend class internal::pipeline_root_task; - friend class pipeline; - friend class thread_bound_filter; - - //! Storage for filter mode and dynamically checked implementation version. - const unsigned char my_filter_mode; - - //! Pointer to previous filter in the pipeline. - filter* prev_filter_in_pipeline; - - //! Pointer to the pipeline. - pipeline* my_pipeline; - - //! Pointer to the next "segment" of filters, or NULL if not required. - /** In each segment, the first filter is not thread-bound but follows a thread-bound one. */ - filter* next_segment; -}; - -//! A stage in a pipeline served by a user thread. -/** @ingroup algorithms */ -class thread_bound_filter: public filter { -public: - enum result_type { - // item was processed - success, - // item is currently not available - item_not_available, - // there are no more items to process - end_of_stream - }; -protected: - explicit thread_bound_filter(mode filter_mode): - filter(static_cast(filter_mode | filter::filter_is_bound)) - { - __TBB_ASSERT(filter_mode & filter::filter_is_serial, "thread-bound filters must be serial"); - } -public: - //! If a data item is available, invoke operator() on that item. - /** This interface is non-blocking. - Returns 'success' if an item was processed. - Returns 'item_not_available' if no item can be processed now - but more may arrive in the future, or if token limit is reached. - Returns 'end_of_stream' if there are no more items to process. */ - result_type __TBB_EXPORTED_METHOD try_process_item(); - - //! Wait until a data item becomes available, and invoke operator() on that item. - /** This interface is blocking. - Returns 'success' if an item was processed. - Returns 'end_of_stream' if there are no more items to process. - Never returns 'item_not_available', as it blocks until another return condition applies. */ - result_type __TBB_EXPORTED_METHOD process_item(); - -private: - //! Internal routine for item processing - result_type internal_process_item(bool is_blocking); -}; - -//! A processing pipeline that applies filters to items. -/** @ingroup algorithms */ -class __TBB_DEPRECATED_MSG("tbb::pipeline is deprecated, use tbb::parallel_pipeline") pipeline { -public: - //! Construct empty pipeline. - __TBB_EXPORTED_METHOD pipeline(); - - /** Though the current implementation declares the destructor virtual, do not rely on this - detail. The virtualness is deprecated and may disappear in future versions of TBB. */ - virtual __TBB_EXPORTED_METHOD ~pipeline(); - - //! Add filter to end of pipeline. - void __TBB_EXPORTED_METHOD add_filter( filter& filter_ ); - - //! Run the pipeline to completion. - void __TBB_EXPORTED_METHOD run( size_t max_number_of_live_tokens ); - -#if __TBB_TASK_GROUP_CONTEXT - //! Run the pipeline to completion with user-supplied context. - void __TBB_EXPORTED_METHOD run( size_t max_number_of_live_tokens, tbb::task_group_context& context ); -#endif - - //! Remove all filters from the pipeline. - void __TBB_EXPORTED_METHOD clear(); - -private: - friend class internal::stage_task; - friend class internal::pipeline_root_task; - friend class filter; - friend class thread_bound_filter; - friend class internal::pipeline_cleaner; - friend class tbb::interface6::internal::pipeline_proxy; - - //! Pointer to first filter in the pipeline. - filter* filter_list; - - //! Pointer to location where address of next filter to be added should be stored. - filter* filter_end; - - //! task who's reference count is used to determine when all stages are done. - task* end_counter; - - //! Number of idle tokens waiting for input stage. - atomic input_tokens; - - //! Global counter of tokens - atomic token_counter; - - //! False until fetch_input returns NULL. - bool end_of_input; - - //! True if the pipeline contains a thread-bound filter; false otherwise. - bool has_thread_bound_filters; - - //! Remove filter from pipeline. - void remove_filter( filter& filter_ ); - - //! Not used, but retained to satisfy old export files. - void __TBB_EXPORTED_METHOD inject_token( task& self ); - -#if __TBB_TASK_GROUP_CONTEXT - //! Does clean up if pipeline is cancelled or exception occurred - void clear_filters(); -#endif -}; - -//------------------------------------------------------------------------ -// Support for lambda-friendly parallel_pipeline interface -//------------------------------------------------------------------------ - -namespace interface6 { - -namespace internal { - template class concrete_filter; -} - -//! input_filter control to signal end-of-input for parallel_pipeline -class flow_control { - bool is_pipeline_stopped; - flow_control() { is_pipeline_stopped = false; } - template friend class internal::concrete_filter; -public: - void stop() { is_pipeline_stopped = true; } -}; - -//! @cond INTERNAL -namespace internal { - -// Emulate std::is_trivially_copyable (false positives not allowed, false negatives suboptimal but safe). -#if __TBB_CPP11_TYPE_PROPERTIES_PRESENT -template struct tbb_trivially_copyable { enum { value = std::is_trivially_copyable::value }; }; -#else -template struct tbb_trivially_copyable { enum { value = false }; }; -template struct tbb_trivially_copyable < T* > { enum { value = true }; }; -template<> struct tbb_trivially_copyable < bool > { enum { value = true }; }; -template<> struct tbb_trivially_copyable < char > { enum { value = true }; }; -template<> struct tbb_trivially_copyable < signed char > { enum { value = true }; }; -template<> struct tbb_trivially_copyable { enum { value = true }; }; -template<> struct tbb_trivially_copyable < short > { enum { value = true }; }; -template<> struct tbb_trivially_copyable { enum { value = true }; }; -template<> struct tbb_trivially_copyable < int > { enum { value = true }; }; -template<> struct tbb_trivially_copyable { enum { value = true }; }; -template<> struct tbb_trivially_copyable < long > { enum { value = true }; }; -template<> struct tbb_trivially_copyable { enum { value = true }; }; -template<> struct tbb_trivially_copyable < long long> { enum { value = true }; }; -template<> struct tbb_trivially_copyable { enum { value = true }; }; -template<> struct tbb_trivially_copyable < float > { enum { value = true }; }; -template<> struct tbb_trivially_copyable < double > { enum { value = true }; }; -template<> struct tbb_trivially_copyable < long double > { enum { value = true }; }; -#if !_MSC_VER || defined(_NATIVE_WCHAR_T_DEFINED) -template<> struct tbb_trivially_copyable < wchar_t > { enum { value = true }; }; -#endif /* _MSC_VER||!defined(_NATIVE_WCHAR_T_DEFINED) */ -#endif // tbb_trivially_copyable - -template -struct use_allocator { - enum { value = sizeof(T) > sizeof(void *) || !tbb_trivially_copyable::value }; -}; - -// A helper class to customize how a type is passed between filters. -// Usage: token_helper::value> -template class token_helper; - -// using tbb_allocator -template -class token_helper { -public: - typedef typename tbb::tbb_allocator allocator; - typedef T* pointer; - typedef T value_type; -#if __TBB_CPP11_RVALUE_REF_PRESENT - static pointer create_token(value_type && source) -#else - static pointer create_token(const value_type & source) -#endif - { - pointer output_t = allocator().allocate(1); - return new (output_t) T(tbb::internal::move(source)); - } - static value_type & token(pointer & t) { return *t; } - static void * cast_to_void_ptr(pointer ref) { return (void *) ref; } - static pointer cast_from_void_ptr(void * ref) { return (pointer)ref; } - static void destroy_token(pointer token) { - allocator().destroy(token); - allocator().deallocate(token,1); - } -}; - -// pointer specialization -template -class token_helper { -public: - typedef T* pointer; - typedef T* value_type; - static pointer create_token(const value_type & source) { return source; } - static value_type & token(pointer & t) { return t; } - static void * cast_to_void_ptr(pointer ref) { return (void *)ref; } - static pointer cast_from_void_ptr(void * ref) { return (pointer)ref; } - static void destroy_token( pointer /*token*/) {} -}; - -// converting type to and from void*, passing objects directly -template -class token_helper { - typedef union { - T actual_value; - void * void_overlay; - } type_to_void_ptr_map; -public: - typedef T pointer; // not really a pointer in this case. - typedef T value_type; - static pointer create_token(const value_type & source) { return source; } - static value_type & token(pointer & t) { return t; } - static void * cast_to_void_ptr(pointer ref) { - type_to_void_ptr_map mymap; - mymap.void_overlay = NULL; - mymap.actual_value = ref; - return mymap.void_overlay; - } - static pointer cast_from_void_ptr(void * ref) { - type_to_void_ptr_map mymap; - mymap.void_overlay = ref; - return mymap.actual_value; - } - static void destroy_token( pointer /*token*/) {} -}; - -// intermediate -template -class concrete_filter: public tbb::filter { - const Body& my_body; - typedef token_helper::value> t_helper; - typedef typename t_helper::pointer t_pointer; - typedef token_helper::value> u_helper; - typedef typename u_helper::pointer u_pointer; - - void* operator()(void* input) __TBB_override { - t_pointer temp_input = t_helper::cast_from_void_ptr(input); - u_pointer output_u = u_helper::create_token(my_body(tbb::internal::move(t_helper::token(temp_input)))); - t_helper::destroy_token(temp_input); - return u_helper::cast_to_void_ptr(output_u); - } - - void finalize(void * input) __TBB_override { - t_pointer temp_input = t_helper::cast_from_void_ptr(input); - t_helper::destroy_token(temp_input); - } - -public: - concrete_filter(tbb::filter::mode filter_mode, const Body& body) : filter(filter_mode), my_body(body) {} -}; - -// input -template -class concrete_filter: public filter { - const Body& my_body; - typedef token_helper::value> u_helper; - typedef typename u_helper::pointer u_pointer; - - void* operator()(void*) __TBB_override { - flow_control control; - u_pointer output_u = u_helper::create_token(my_body(control)); - if(control.is_pipeline_stopped) { - u_helper::destroy_token(output_u); - set_end_of_input(); - return NULL; - } - return u_helper::cast_to_void_ptr(output_u); - } - -public: - concrete_filter(tbb::filter::mode filter_mode, const Body& body) : - filter(static_cast(filter_mode | filter_may_emit_null)), - my_body(body) - {} -}; - -// output -template -class concrete_filter: public filter { - const Body& my_body; - typedef token_helper::value> t_helper; - typedef typename t_helper::pointer t_pointer; - - void* operator()(void* input) __TBB_override { - t_pointer temp_input = t_helper::cast_from_void_ptr(input); - my_body(tbb::internal::move(t_helper::token(temp_input))); - t_helper::destroy_token(temp_input); - return NULL; - } - void finalize(void* input) __TBB_override { - t_pointer temp_input = t_helper::cast_from_void_ptr(input); - t_helper::destroy_token(temp_input); - } - -public: - concrete_filter(tbb::filter::mode filter_mode, const Body& body) : filter(filter_mode), my_body(body) {} -}; - -template -class concrete_filter: public filter { - const Body& my_body; - - void* operator()(void*) __TBB_override { - flow_control control; - my_body(control); - void* output = control.is_pipeline_stopped ? NULL : (void*)(intptr_t)-1; - return output; - } -public: - concrete_filter(filter::mode filter_mode, const Body& body) : filter(filter_mode), my_body(body) {} -}; - -//! The class that represents an object of the pipeline for parallel_pipeline(). -/** It primarily serves as RAII class that deletes heap-allocated filter instances. */ -class pipeline_proxy { - tbb::pipeline my_pipe; -public: - pipeline_proxy( const filter_t& filter_chain ); - ~pipeline_proxy() { - while( filter* f = my_pipe.filter_list ) - delete f; // filter destructor removes it from the pipeline - } - tbb::pipeline* operator->() { return &my_pipe; } -}; - -//! Abstract base class that represents a node in a parse tree underlying a filter_t. -/** These nodes are always heap-allocated and can be shared by filter_t objects. */ -class filter_node: tbb::internal::no_copy { - /** Count must be atomic because it is hidden state for user, but might be shared by threads. */ - tbb::atomic ref_count; -protected: - filter_node() { - ref_count = 0; -#ifdef __TBB_TEST_FILTER_NODE_COUNT - ++(__TBB_TEST_FILTER_NODE_COUNT); -#endif - } -public: - //! Add concrete_filter to pipeline - virtual void add_to( pipeline& ) = 0; - //! Increment reference count - void add_ref() { ++ref_count; } - //! Decrement reference count and delete if it becomes zero. - void remove_ref() { - __TBB_ASSERT(ref_count>0,"ref_count underflow"); - if( --ref_count==0 ) - delete this; - } - virtual ~filter_node() { -#ifdef __TBB_TEST_FILTER_NODE_COUNT - --(__TBB_TEST_FILTER_NODE_COUNT); -#endif - } -}; - -//! Node in parse tree representing result of make_filter. -template -class filter_node_leaf: public filter_node { - const tbb::filter::mode mode; - const Body body; - void add_to( pipeline& p ) __TBB_override { - concrete_filter* f = new concrete_filter(mode,body); - p.add_filter( *f ); - } -public: - filter_node_leaf( tbb::filter::mode m, const Body& b ) : mode(m), body(b) {} -}; - -//! Node in parse tree representing join of two filters. -class filter_node_join: public filter_node { - friend class filter_node; // to suppress GCC 3.2 warnings - filter_node& left; - filter_node& right; - ~filter_node_join() { - left.remove_ref(); - right.remove_ref(); - } - void add_to( pipeline& p ) __TBB_override { - left.add_to(p); - right.add_to(p); - } -public: - filter_node_join( filter_node& x, filter_node& y ) : left(x), right(y) { - left.add_ref(); - right.add_ref(); - } -}; - -} // namespace internal -//! @endcond - -//! Create a filter to participate in parallel_pipeline -template -filter_t make_filter(tbb::filter::mode mode, const Body& body) { - return new internal::filter_node_leaf(mode, body); -} - -template -filter_t operator& (const filter_t& left, const filter_t& right) { - __TBB_ASSERT(left.root,"cannot use default-constructed filter_t as left argument of '&'"); - __TBB_ASSERT(right.root,"cannot use default-constructed filter_t as right argument of '&'"); - return new internal::filter_node_join(*left.root,*right.root); -} - -//! Class representing a chain of type-safe pipeline filters -template -class filter_t { - typedef internal::filter_node filter_node; - filter_node* root; - filter_t( filter_node* root_ ) : root(root_) { - root->add_ref(); - } - friend class internal::pipeline_proxy; - template - friend filter_t make_filter(tbb::filter::mode, const Body& ); - template - friend filter_t operator& (const filter_t& , const filter_t& ); -public: - // TODO: add move-constructors, move-assignment, etc. where C++11 is available. - filter_t() : root(NULL) {} - filter_t( const filter_t& rhs ) : root(rhs.root) { - if( root ) root->add_ref(); - } - template - filter_t( tbb::filter::mode mode, const Body& body ) : - root( new internal::filter_node_leaf(mode, body) ) { - root->add_ref(); - } - - void operator=( const filter_t& rhs ) { - // Order of operations below carefully chosen so that reference counts remain correct - // in unlikely event that remove_ref throws exception. - filter_node* old = root; - root = rhs.root; - if( root ) root->add_ref(); - if( old ) old->remove_ref(); - } - ~filter_t() { - if( root ) root->remove_ref(); - } - void clear() { - // Like operator= with filter_t() on right side. - if( root ) { - filter_node* old = root; - root = NULL; - old->remove_ref(); - } - } -}; - -inline internal::pipeline_proxy::pipeline_proxy( const filter_t& filter_chain ) : my_pipe() { - __TBB_ASSERT( filter_chain.root, "cannot apply parallel_pipeline to default-constructed filter_t" ); - filter_chain.root->add_to(my_pipe); -} - -inline void parallel_pipeline(size_t max_number_of_live_tokens, const filter_t& filter_chain -#if __TBB_TASK_GROUP_CONTEXT - , tbb::task_group_context& context -#endif - ) { - internal::pipeline_proxy pipe(filter_chain); - // tbb::pipeline::run() is called via the proxy - pipe->run(max_number_of_live_tokens -#if __TBB_TASK_GROUP_CONTEXT - , context -#endif - ); -} - -#if __TBB_TASK_GROUP_CONTEXT -inline void parallel_pipeline(size_t max_number_of_live_tokens, const filter_t& filter_chain) { - tbb::task_group_context context; - parallel_pipeline(max_number_of_live_tokens, filter_chain, context); -} -#endif // __TBB_TASK_GROUP_CONTEXT - -} // interface6 - -using interface6::flow_control; -using interface6::filter_t; -using interface6::make_filter; -using interface6::parallel_pipeline; - -} // tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_pipeline_H_include_area - -#endif /* __TBB_pipeline_H */ diff --git a/src/3rd party/tbb/profiling.h b/src/3rd party/tbb/profiling.h new file mode 100644 index 0000000000..c7cea9c590 --- /dev/null +++ b/src/3rd party/tbb/profiling.h @@ -0,0 +1,17 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "../oneapi/tbb/profiling.h" diff --git a/src/3rd party/tbb/queuing_mutex.h b/src/3rd party/tbb/queuing_mutex.h index 17a858fa5b..ad031e4eb7 100644 --- a/src/3rd party/tbb/queuing_mutex.h +++ b/src/3rd party/tbb/queuing_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,100 +14,4 @@ limitations under the License. */ -#ifndef __TBB_queuing_mutex_H -#define __TBB_queuing_mutex_H - -#define __TBB_queuing_mutex_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include -#include "atomic.h" -#include "tbb_profiling.h" - -namespace tbb { - -//! Queuing mutex with local-only spinning. -/** @ingroup synchronization */ -class queuing_mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Construct unacquired mutex. - queuing_mutex() { - q_tail = NULL; -#if TBB_USE_THREADING_TOOLS - internal_construct(); -#endif - } - - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - class scoped_lock: internal::no_copy { - //! Initialize fields to mean "no lock held". - void initialize() { - mutex = NULL; - going = 0; -#if TBB_USE_ASSERT - internal::poison_pointer(next); -#endif /* TBB_USE_ASSERT */ - } - - public: - //! Construct lock that has not acquired a mutex. - /** Equivalent to zero-initialization of *this. */ - scoped_lock() {initialize();} - - //! Acquire lock on given mutex. - scoped_lock( queuing_mutex& m ) { - initialize(); - acquire(m); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if( mutex ) release(); - } - - //! Acquire lock on given mutex. - void __TBB_EXPORTED_METHOD acquire( queuing_mutex& m ); - - //! Acquire lock on given mutex if free (i.e. non-blocking) - bool __TBB_EXPORTED_METHOD try_acquire( queuing_mutex& m ); - - //! Release lock. - void __TBB_EXPORTED_METHOD release(); - - private: - //! The pointer to the mutex owned, or NULL if not holding a mutex. - queuing_mutex* mutex; - - //! The pointer to the next competitor for a mutex - scoped_lock *next; - - //! The local spin-wait variable - /** Inverted (0 - blocked, 1 - acquired the mutex) for the sake of - zero-initialization. Defining it as an entire word instead of - a byte seems to help performance slightly. */ - uintptr_t going; - }; - - void __TBB_EXPORTED_METHOD internal_construct(); - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = true; - -private: - //! The last competitor requesting the lock - atomic q_tail; - -}; - -__TBB_DEFINE_PROFILING_SET_NAME(queuing_mutex) - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_queuing_mutex_H_include_area - -#endif /* __TBB_queuing_mutex_H */ +#include "../oneapi/tbb/queuing_mutex.h" diff --git a/src/3rd party/tbb/queuing_rw_mutex.h b/src/3rd party/tbb/queuing_rw_mutex.h index 86f1b8fd62..203727ccc5 100644 --- a/src/3rd party/tbb/queuing_rw_mutex.h +++ b/src/3rd party/tbb/queuing_rw_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,141 +14,4 @@ limitations under the License. */ -#ifndef __TBB_queuing_rw_mutex_H -#define __TBB_queuing_rw_mutex_H - -#define __TBB_queuing_rw_mutex_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include -#include "atomic.h" -#include "tbb_profiling.h" - -namespace tbb { - -//! Queuing reader-writer mutex with local-only spinning. -/** Adapted from Krieger, Stumm, et al. pseudocode at - http://www.eecg.toronto.edu/parallel/pubs_abs.html#Krieger_etal_ICPP93 - @ingroup synchronization */ -class queuing_rw_mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Construct unacquired mutex. - queuing_rw_mutex() { - q_tail = NULL; -#if TBB_USE_THREADING_TOOLS - internal_construct(); -#endif - } - - //! Destructor asserts if the mutex is acquired, i.e. q_tail is non-NULL - ~queuing_rw_mutex() { -#if TBB_USE_ASSERT - __TBB_ASSERT( !q_tail, "destruction of an acquired mutex"); -#endif - } - - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - class scoped_lock: internal::no_copy { - //! Initialize fields to mean "no lock held". - void initialize() { - my_mutex = NULL; - my_internal_lock = 0; - my_going = 0; -#if TBB_USE_ASSERT - my_state = 0xFF; // Set to invalid state - internal::poison_pointer(my_next); - internal::poison_pointer(my_prev); -#endif /* TBB_USE_ASSERT */ - } - - public: - //! Construct lock that has not acquired a mutex. - /** Equivalent to zero-initialization of *this. */ - scoped_lock() {initialize();} - - //! Acquire lock on given mutex. - scoped_lock( queuing_rw_mutex& m, bool write=true ) { - initialize(); - acquire(m,write); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if( my_mutex ) release(); - } - - //! Acquire lock on given mutex. - void acquire( queuing_rw_mutex& m, bool write=true ); - - //! Acquire lock on given mutex if free (i.e. non-blocking) - bool try_acquire( queuing_rw_mutex& m, bool write=true ); - - //! Release lock. - void release(); - - //! Upgrade reader to become a writer. - /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ - bool upgrade_to_writer(); - - //! Downgrade writer to become a reader. - bool downgrade_to_reader(); - - private: - //! The pointer to the mutex owned, or NULL if not holding a mutex. - queuing_rw_mutex* my_mutex; - - //! The pointer to the previous and next competitors for a mutex - scoped_lock *__TBB_atomic my_prev, *__TBB_atomic my_next; - - typedef unsigned char state_t; - - //! State of the request: reader, writer, active reader, other service states - atomic my_state; - - //! The local spin-wait variable - /** Corresponds to "spin" in the pseudocode but inverted for the sake of zero-initialization */ - unsigned char __TBB_atomic my_going; - - //! A tiny internal lock - unsigned char my_internal_lock; - - //! Acquire the internal lock - void acquire_internal_lock(); - - //! Try to acquire the internal lock - /** Returns true if lock was successfully acquired. */ - bool try_acquire_internal_lock(); - - //! Release the internal lock - void release_internal_lock(); - - //! Wait for internal lock to be released - void wait_for_release_of_internal_lock(); - - //! A helper function - void unblock_or_wait_on_internal_lock( uintptr_t ); - }; - - void __TBB_EXPORTED_METHOD internal_construct(); - - // Mutex traits - static const bool is_rw_mutex = true; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = true; - -private: - //! The last competitor requesting the lock - atomic q_tail; - -}; - -__TBB_DEFINE_PROFILING_SET_NAME(queuing_rw_mutex) - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_queuing_rw_mutex_H_include_area - -#endif /* __TBB_queuing_rw_mutex_H */ +#include "../oneapi/tbb/queuing_rw_mutex.h" diff --git a/src/3rd party/tbb/reader_writer_lock.h b/src/3rd party/tbb/reader_writer_lock.h deleted file mode 100644 index 0783c79212..0000000000 --- a/src/3rd party/tbb/reader_writer_lock.h +++ /dev/null @@ -1,246 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include "internal/_deprecated_header_message_guard.h" - -#if !defined(__TBB_show_deprecation_message_reader_writer_lock_H) && defined(__TBB_show_deprecated_header_message) -#define __TBB_show_deprecation_message_reader_writer_lock_H -#pragma message("TBB Warning: tbb/reader_writer_lock.h is deprecated. For details, please see Deprecated Features appendix in the TBB reference manual.") -#endif - -#if defined(__TBB_show_deprecated_header_message) -#undef __TBB_show_deprecated_header_message -#endif - -#ifndef __TBB_reader_writer_lock_H -#define __TBB_reader_writer_lock_H - -#define __TBB_reader_writer_lock_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "tbb_thread.h" -#include "tbb_allocator.h" -#include "atomic.h" - -namespace tbb { -namespace interface5 { -//! Writer-preference reader-writer lock with local-only spinning on readers. -/** Loosely adapted from Mellor-Crummey and Scott pseudocode at - http://www.cs.rochester.edu/research/synchronization/pseudocode/rw.html#s_wp - @ingroup synchronization */ - class __TBB_DEPRECATED_VERBOSE_MSG("tbb::reader_writer_lock is deprecated, use std::shared_mutex") - reader_writer_lock : tbb::internal::no_copy { - public: - friend class scoped_lock; - friend class scoped_lock_read; - //! Status type for nodes associated with lock instances - /** waiting_nonblocking: the wait state for nonblocking lock - instances; for writes, these transition straight to active - states; for reads, these are unused. - - waiting: the start and spin state for all lock instances; these will - transition to active state when appropriate. Non-blocking write locks - transition from this state to waiting_nonblocking immediately. - - active: the active state means that the lock instance holds - the lock; it will transition to invalid state during node deletion - - invalid: the end state for all nodes; this is set in the - destructor so if we encounter this state, we are looking at - memory that has already been freed - - The state diagrams below describe the status transitions. - Single arrows indicate that the thread that owns the node is - responsible for the transition; double arrows indicate that - any thread could make the transition. - - State diagram for scoped_lock status: - - waiting ----------> waiting_nonblocking - | _____________/ | - V V V - active -----------------> invalid - - State diagram for scoped_lock_read status: - - waiting - | - V - active ----------------->invalid - - */ - enum status_t { waiting_nonblocking, waiting, active, invalid }; - - //! Constructs a new reader_writer_lock - reader_writer_lock() { - internal_construct(); - } - - //! Destructs a reader_writer_lock object - ~reader_writer_lock() { - internal_destroy(); - } - - //! The scoped lock pattern for write locks - /** Scoped locks help avoid the common problem of forgetting to release the lock. - This type also serves as the node for queuing locks. */ - class scoped_lock : tbb::internal::no_copy { - public: - friend class reader_writer_lock; - - //! Construct with blocking attempt to acquire write lock on the passed-in lock - scoped_lock(reader_writer_lock& lock) { - internal_construct(lock); - } - - //! Destructor, releases the write lock - ~scoped_lock() { - internal_destroy(); - } - - void* operator new(size_t s) { - return tbb::internal::allocate_via_handler_v3(s); - } - void operator delete(void* p) { - tbb::internal::deallocate_via_handler_v3(p); - } - - private: - //! The pointer to the mutex to lock - reader_writer_lock *mutex; - //! The next queued competitor for the mutex - scoped_lock* next; - //! Status flag of the thread associated with this node - atomic status; - - //! Construct scoped_lock that is not holding lock - scoped_lock(); - - void __TBB_EXPORTED_METHOD internal_construct(reader_writer_lock&); - void __TBB_EXPORTED_METHOD internal_destroy(); - }; - - //! The scoped lock pattern for read locks - class scoped_lock_read : tbb::internal::no_copy { - public: - friend class reader_writer_lock; - - //! Construct with blocking attempt to acquire read lock on the passed-in lock - scoped_lock_read(reader_writer_lock& lock) { - internal_construct(lock); - } - - //! Destructor, releases the read lock - ~scoped_lock_read() { - internal_destroy(); - } - - void* operator new(size_t s) { - return tbb::internal::allocate_via_handler_v3(s); - } - void operator delete(void* p) { - tbb::internal::deallocate_via_handler_v3(p); - } - - private: - //! The pointer to the mutex to lock - reader_writer_lock *mutex; - //! The next queued competitor for the mutex - scoped_lock_read *next; - //! Status flag of the thread associated with this node - atomic status; - - //! Construct scoped_lock_read that is not holding lock - scoped_lock_read(); - - void __TBB_EXPORTED_METHOD internal_construct(reader_writer_lock&); - void __TBB_EXPORTED_METHOD internal_destroy(); - }; - - //! Acquires the reader_writer_lock for write. - /** If the lock is currently held in write mode by another - context, the writer will block by spinning on a local - variable. Exceptions thrown: improper_lock The context tries - to acquire a reader_writer_lock that it already has write - ownership of.*/ - void __TBB_EXPORTED_METHOD lock(); - - //! Tries to acquire the reader_writer_lock for write. - /** This function does not block. Return Value: True or false, - depending on whether the lock is acquired or not. If the lock - is already held by this acquiring context, try_lock() returns - false. */ - bool __TBB_EXPORTED_METHOD try_lock(); - - //! Acquires the reader_writer_lock for read. - /** If the lock is currently held by a writer, this reader will - block and wait until the writers are done. Exceptions thrown: - improper_lock The context tries to acquire a - reader_writer_lock that it already has write ownership of. */ - void __TBB_EXPORTED_METHOD lock_read(); - - //! Tries to acquire the reader_writer_lock for read. - /** This function does not block. Return Value: True or false, - depending on whether the lock is acquired or not. */ - bool __TBB_EXPORTED_METHOD try_lock_read(); - - //! Releases the reader_writer_lock - void __TBB_EXPORTED_METHOD unlock(); - - private: - void __TBB_EXPORTED_METHOD internal_construct(); - void __TBB_EXPORTED_METHOD internal_destroy(); - - //! Attempts to acquire write lock - /** If unavailable, spins in blocking case, returns false in non-blocking case. */ - bool start_write(scoped_lock *); - //! Sets writer_head to w and attempts to unblock - void set_next_writer(scoped_lock *w); - //! Relinquishes write lock to next waiting writer or group of readers - void end_write(scoped_lock *); - //! Checks if current thread holds write lock - bool is_current_writer(); - - //! Attempts to acquire read lock - /** If unavailable, spins in blocking case, returns false in non-blocking case. */ - void start_read(scoped_lock_read *); - //! Unblocks pending readers - void unblock_readers(); - //! Relinquishes read lock by decrementing counter; last reader wakes pending writer - void end_read(); - - //! The list of pending readers - atomic reader_head; - //! The list of pending writers - atomic writer_head; - //! The last node in the list of pending writers - atomic writer_tail; - //! Writer that owns the mutex; tbb_thread::id() otherwise. - tbb_thread::id my_current_writer; - //! Status of mutex - atomic rdr_count_and_flags; // used with __TBB_AtomicOR, which assumes uintptr_t -}; - -} // namespace interface5 - -using interface5::reader_writer_lock; - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_reader_writer_lock_H_include_area - -#endif /* __TBB_reader_writer_lock_H */ diff --git a/src/3rd party/tbb/recursive_mutex.h b/src/3rd party/tbb/recursive_mutex.h deleted file mode 100644 index 3a3f979426..0000000000 --- a/src/3rd party/tbb/recursive_mutex.h +++ /dev/null @@ -1,248 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include "internal/_deprecated_header_message_guard.h" - -#if !defined(__TBB_show_deprecation_message_recursive_mutex_H) && defined(__TBB_show_deprecated_header_message) -#define __TBB_show_deprecation_message_recursive_mutex_H -#pragma message("TBB Warning: tbb/recursive_mutex.h is deprecated. For details, please see Deprecated Features appendix in the TBB reference manual.") -#endif - -#if defined(__TBB_show_deprecated_header_message) -#undef __TBB_show_deprecated_header_message -#endif - -#ifndef __TBB_recursive_mutex_H -#define __TBB_recursive_mutex_H - -#define __TBB_recursive_mutex_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#else -#include -#endif /* _WIN32||_WIN64 */ - -#include -#include "aligned_space.h" -#include "tbb_stddef.h" -#include "tbb_profiling.h" - -namespace tbb { -//! Mutex that allows recursive mutex acquisition. -/** Mutex that allows recursive mutex acquisition. - @ingroup synchronization */ -class __TBB_DEPRECATED_VERBOSE_MSG("tbb::recursive_mutex is deprecated, use std::recursive_mutex") -recursive_mutex : internal::mutex_copy_deprecated_and_disabled { -public: - //! Construct unacquired recursive_mutex. - recursive_mutex() { -#if TBB_USE_ASSERT || TBB_USE_THREADING_TOOLS - internal_construct(); -#else - #if _WIN32||_WIN64 - InitializeCriticalSectionEx(&impl, 4000, 0); - #else - pthread_mutexattr_t mtx_attr; - int error_code = pthread_mutexattr_init( &mtx_attr ); - if( error_code ) - tbb::internal::handle_perror(error_code,"recursive_mutex: pthread_mutexattr_init failed"); - - pthread_mutexattr_settype( &mtx_attr, PTHREAD_MUTEX_RECURSIVE ); - error_code = pthread_mutex_init( &impl, &mtx_attr ); - if( error_code ) - tbb::internal::handle_perror(error_code,"recursive_mutex: pthread_mutex_init failed"); - - pthread_mutexattr_destroy( &mtx_attr ); - #endif /* _WIN32||_WIN64*/ -#endif /* TBB_USE_ASSERT */ - }; - - ~recursive_mutex() { -#if TBB_USE_ASSERT - internal_destroy(); -#else - #if _WIN32||_WIN64 - DeleteCriticalSection(&impl); - #else - pthread_mutex_destroy(&impl); - - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - }; - - class scoped_lock; - friend class scoped_lock; - - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - class scoped_lock: internal::no_copy { - public: - //! Construct lock that has not acquired a recursive_mutex. - scoped_lock() : my_mutex(NULL) {}; - - //! Acquire lock on given mutex. - scoped_lock( recursive_mutex& mutex ) { -#if TBB_USE_ASSERT - my_mutex = &mutex; -#endif /* TBB_USE_ASSERT */ - acquire( mutex ); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if( my_mutex ) - release(); - } - - //! Acquire lock on given mutex. - void acquire( recursive_mutex& mutex ) { -#if TBB_USE_ASSERT - internal_acquire( mutex ); -#else - my_mutex = &mutex; - mutex.lock(); -#endif /* TBB_USE_ASSERT */ - } - - //! Try acquire lock on given recursive_mutex. - bool try_acquire( recursive_mutex& mutex ) { -#if TBB_USE_ASSERT - return internal_try_acquire( mutex ); -#else - bool result = mutex.try_lock(); - if( result ) - my_mutex = &mutex; - return result; -#endif /* TBB_USE_ASSERT */ - } - - //! Release lock - void release() { -#if TBB_USE_ASSERT - internal_release(); -#else - my_mutex->unlock(); - my_mutex = NULL; -#endif /* TBB_USE_ASSERT */ - } - - private: - //! The pointer to the current recursive_mutex to work - recursive_mutex* my_mutex; - - //! All checks from acquire using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_acquire( recursive_mutex& m ); - - //! All checks from try_acquire using mutex.state were moved here - bool __TBB_EXPORTED_METHOD internal_try_acquire( recursive_mutex& m ); - - //! All checks from release using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_release(); - - friend class recursive_mutex; - }; - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = true; - static const bool is_fair_mutex = false; - - // C++0x compatibility interface - - //! Acquire lock - void lock() { -#if TBB_USE_ASSERT - aligned_space tmp; - new(tmp.begin()) scoped_lock(*this); -#else - #if _WIN32||_WIN64 - EnterCriticalSection(&impl); - #else - int error_code = pthread_mutex_lock(&impl); - if( error_code ) - tbb::internal::handle_perror(error_code,"recursive_mutex: pthread_mutex_lock failed"); - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_lock() { -#if TBB_USE_ASSERT - aligned_space tmp; - return (new(tmp.begin()) scoped_lock)->internal_try_acquire(*this); -#else - #if _WIN32||_WIN64 - return TryEnterCriticalSection(&impl)!=0; - #else - return pthread_mutex_trylock(&impl)==0; - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Release lock - void unlock() { -#if TBB_USE_ASSERT - aligned_space tmp; - scoped_lock& s = *tmp.begin(); - s.my_mutex = this; - s.internal_release(); -#else - #if _WIN32||_WIN64 - LeaveCriticalSection(&impl); - #else - pthread_mutex_unlock(&impl); - #endif /* _WIN32||_WIN64 */ -#endif /* TBB_USE_ASSERT */ - } - - //! Return native_handle - #if _WIN32||_WIN64 - typedef LPCRITICAL_SECTION native_handle_type; - #else - typedef pthread_mutex_t* native_handle_type; - #endif - native_handle_type native_handle() { return (native_handle_type) &impl; } - -private: -#if _WIN32||_WIN64 - CRITICAL_SECTION impl; - enum state_t { - INITIALIZED=0x1234, - DESTROYED=0x789A, - } state; -#else - pthread_mutex_t impl; -#endif /* _WIN32||_WIN64 */ - - //! All checks from mutex constructor using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_construct(); - - //! All checks from mutex destructor using mutex.state were moved here - void __TBB_EXPORTED_METHOD internal_destroy(); -}; - -__TBB_DEFINE_PROFILING_SET_NAME(recursive_mutex) - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_recursive_mutex_H_include_area - -#endif /* __TBB_recursive_mutex_H */ diff --git a/src/3rd party/tbb/runtime_loader.h b/src/3rd party/tbb/runtime_loader.h deleted file mode 100644 index 2c82b7089c..0000000000 --- a/src/3rd party/tbb/runtime_loader.h +++ /dev/null @@ -1,193 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include "internal/_deprecated_header_message_guard.h" - -#if !defined(__TBB_show_deprecation_message_runtime_loader_H) && defined(__TBB_show_deprecated_header_message) -#define __TBB_show_deprecation_message_runtime_loader_H -#pragma message("TBB Warning: tbb/runtime_loader.h is deprecated. For details, please see Deprecated Features appendix in the TBB reference manual.") -#endif - -#if defined(__TBB_show_deprecated_header_message) -#undef __TBB_show_deprecated_header_message -#endif - -#ifndef __TBB_runtime_loader_H -#define __TBB_runtime_loader_H - -#define __TBB_runtime_loader_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#if ! TBB_PREVIEW_RUNTIME_LOADER - #error Set TBB_PREVIEW_RUNTIME_LOADER to include runtime_loader.h -#endif - -#include "tbb_stddef.h" -#include - -#if _MSC_VER - #if ! __TBB_NO_IMPLICIT_LINKAGE - #ifdef _DEBUG - #pragma comment( linker, "/nodefaultlib:tbb_debug.lib" ) - #pragma comment( linker, "/defaultlib:tbbproxy_debug.lib" ) - #else - #pragma comment( linker, "/nodefaultlib:tbb.lib" ) - #pragma comment( linker, "/defaultlib:tbbproxy.lib" ) - #endif - #endif -#endif - -namespace tbb { - -namespace interface6 { - -//! Load TBB at runtime. -/*! - -\b Usage: - -In source code: - -\code -#include "tbb/runtime_loader.h" - -char const * path[] = { "/lib/ia32", NULL }; -tbb::runtime_loader loader( path ); - -// Now use TBB. -\endcode - -Link with \c tbbproxy.lib (or \c libtbbproxy.a) instead of \c tbb.lib (\c libtbb.dylib, -\c libtbb.so). - -TBB library will be loaded at runtime from \c /lib/ia32 directory. - -\b Attention: - -All \c runtime_loader objects (in the same module, i.e. exe or dll) share some global state. -The most noticeable piece of global state is loaded TBB library. -There are some implications: - - - Only one TBB library can be loaded per module. - - - If one object has already loaded TBB library, another object will not load TBB. - If the loaded TBB library is suitable for the second object, both will use TBB - cooperatively, otherwise the second object will report an error. - - - \c runtime_loader objects will not work (correctly) in parallel due to absence of - synchronization. - -*/ - -class __TBB_DEPRECATED_VERBOSE runtime_loader : tbb::internal::no_copy { - - public: - - //! Error mode constants. - enum error_mode { - em_status, //!< Save status of operation and continue. - em_throw, //!< Throw an exception of tbb::runtime_loader::error_code type. - em_abort //!< Print message to \c stderr and call \c abort(). - }; // error_mode - - //! Error codes. - enum error_code { - ec_ok, //!< No errors. - ec_bad_call, //!< Invalid function call (e. g. load() called when TBB is already loaded). - ec_bad_arg, //!< Invalid argument passed. - ec_bad_lib, //!< Invalid library found (e. g. \c TBB_runtime_version symbol not found). - ec_bad_ver, //!< TBB found but version is not suitable. - ec_no_lib //!< No suitable TBB library found. - }; // error_code - - //! Initialize object but do not load TBB. - runtime_loader( error_mode mode = em_abort ); - - //! Initialize object and load TBB. - /*! - See load() for details. - - If error mode is \c em_status, call status() to check whether TBB was loaded or not. - */ - runtime_loader( - char const * path[], //!< List of directories to search TBB in. - int min_ver = TBB_INTERFACE_VERSION, //!< Minimal suitable version of TBB. - int max_ver = INT_MAX, //!< Maximal suitable version of TBB. - error_mode mode = em_abort //!< Error mode for this object. - ); - - //! Destroy object. - ~runtime_loader(); - - //! Load TBB. - /*! - The method searches the directories specified in \c path[] array for the TBB library. - When the library is found, it is loaded and its version is checked. If the version is - not suitable, the library is unloaded, and the search continues. - - \b Note: - - For security reasons, avoid using relative directory names. For example, never load - TBB from current (\c "."), parent (\c "..") or any other relative directory (like - \c "lib" ). Use only absolute directory names (e. g. "/usr/local/lib"). - - For the same security reasons, avoid using system default directories (\c "") on - Windows. (See http://www.microsoft.com/technet/security/advisory/2269637.mspx for - details.) - - Neglecting these rules may cause your program to execute 3-rd party malicious code. - - \b Errors: - - \c ec_bad_call - TBB already loaded by this object. - - \c ec_bad_arg - \p min_ver and/or \p max_ver negative or zero, - or \p min_ver > \p max_ver. - - \c ec_bad_ver - TBB of unsuitable version already loaded by another object. - - \c ec_no_lib - No suitable library found. - */ - error_code - load( - char const * path[], //!< List of directories to search TBB in. - int min_ver = TBB_INTERFACE_VERSION, //!< Minimal suitable version of TBB. - int max_ver = INT_MAX //!< Maximal suitable version of TBB. - - ); - - - //! Report status. - /*! - If error mode is \c em_status, the function returns status of the last operation. - */ - error_code status(); - - private: - - error_mode const my_mode; - error_code my_status; - bool my_loaded; - -}; // class runtime_loader - -} // namespace interface6 - -using interface6::runtime_loader; - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_runtime_loader_H_include_area - -#endif /* __TBB_runtime_loader_H */ - diff --git a/src/3rd party/tbb/rw_mutex.h b/src/3rd party/tbb/rw_mutex.h new file mode 100644 index 0000000000..f2499ebace --- /dev/null +++ b/src/3rd party/tbb/rw_mutex.h @@ -0,0 +1,17 @@ +/* + Copyright (c) 2023 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "../oneapi/tbb/rw_mutex.h" diff --git a/src/3rd party/tbb/scalable_allocator.h b/src/3rd party/tbb/scalable_allocator.h index a077e00c14..5c654ebd68 100644 --- a/src/3rd party/tbb/scalable_allocator.h +++ b/src/3rd party/tbb/scalable_allocator.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,375 +14,4 @@ limitations under the License. */ -#ifndef __TBB_scalable_allocator_H -#define __TBB_scalable_allocator_H -/** @file */ - -#include /* Need ptrdiff_t and size_t from here. */ -#if !_MSC_VER -#include /* Need intptr_t from here. */ -#endif - -#if !defined(__cplusplus) && __ICC==1100 - #pragma warning (push) - #pragma warning (disable: 991) -#endif - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -#if _MSC_VER >= 1400 -#define __TBB_EXPORTED_FUNC __cdecl -#else -#define __TBB_EXPORTED_FUNC -#endif - -/** The "malloc" analogue to allocate block of memory of size bytes. - * @ingroup memory_allocation */ -void * __TBB_EXPORTED_FUNC scalable_malloc (size_t size); - -/** The "free" analogue to discard a previously allocated piece of memory. - @ingroup memory_allocation */ -void __TBB_EXPORTED_FUNC scalable_free (void* ptr); - -/** The "realloc" analogue complementing scalable_malloc. - @ingroup memory_allocation */ -void * __TBB_EXPORTED_FUNC scalable_realloc (void* ptr, size_t size); - -/** The "calloc" analogue complementing scalable_malloc. - @ingroup memory_allocation */ -void * __TBB_EXPORTED_FUNC scalable_calloc (size_t nobj, size_t size); - -/** The "posix_memalign" analogue. - @ingroup memory_allocation */ -int __TBB_EXPORTED_FUNC scalable_posix_memalign (void** memptr, size_t alignment, size_t size); - -/** The "_aligned_malloc" analogue. - @ingroup memory_allocation */ -void * __TBB_EXPORTED_FUNC scalable_aligned_malloc (size_t size, size_t alignment); - -/** The "_aligned_realloc" analogue. - @ingroup memory_allocation */ -void * __TBB_EXPORTED_FUNC scalable_aligned_realloc (void* ptr, size_t size, size_t alignment); - -/** The "_aligned_free" analogue. - @ingroup memory_allocation */ -void __TBB_EXPORTED_FUNC scalable_aligned_free (void* ptr); - -/** The analogue of _msize/malloc_size/malloc_usable_size. - Returns the usable size of a memory block previously allocated by scalable_*, - or 0 (zero) if ptr does not point to such a block. - @ingroup memory_allocation */ -size_t __TBB_EXPORTED_FUNC scalable_msize (void* ptr); - -/* Results for scalable_allocation_* functions */ -typedef enum { - TBBMALLOC_OK, - TBBMALLOC_INVALID_PARAM, - TBBMALLOC_UNSUPPORTED, - TBBMALLOC_NO_MEMORY, - TBBMALLOC_NO_EFFECT -} ScalableAllocationResult; - -/* Setting TBB_MALLOC_USE_HUGE_PAGES environment variable to 1 enables huge pages. - scalable_allocation_mode call has priority over environment variable. */ -typedef enum { - TBBMALLOC_USE_HUGE_PAGES, /* value turns using huge pages on and off */ - /* deprecated, kept for backward compatibility only */ - USE_HUGE_PAGES = TBBMALLOC_USE_HUGE_PAGES, - /* try to limit memory consumption value (Bytes), clean internal buffers - if limit is exceeded, but not prevents from requesting memory from OS */ - TBBMALLOC_SET_SOFT_HEAP_LIMIT, - /* Lower bound for the size (Bytes), that is interpreted as huge - * and not released during regular cleanup operations. */ - TBBMALLOC_SET_HUGE_SIZE_THRESHOLD -} AllocationModeParam; - -/** Set TBB allocator-specific allocation modes. - @ingroup memory_allocation */ -int __TBB_EXPORTED_FUNC scalable_allocation_mode(int param, intptr_t value); - -typedef enum { - /* Clean internal allocator buffers for all threads. - Returns TBBMALLOC_NO_EFFECT if no buffers cleaned, - TBBMALLOC_OK if some memory released from buffers. */ - TBBMALLOC_CLEAN_ALL_BUFFERS, - /* Clean internal allocator buffer for current thread only. - Return values same as for TBBMALLOC_CLEAN_ALL_BUFFERS. */ - TBBMALLOC_CLEAN_THREAD_BUFFERS -} ScalableAllocationCmd; - -/** Call TBB allocator-specific commands. - @ingroup memory_allocation */ -int __TBB_EXPORTED_FUNC scalable_allocation_command(int cmd, void *param); - -#ifdef __cplusplus -} /* extern "C" */ -#endif /* __cplusplus */ - -#ifdef __cplusplus - -//! The namespace rml contains components of low-level memory pool interface. -namespace rml { -class MemoryPool; - -typedef void *(*rawAllocType)(intptr_t pool_id, size_t &bytes); -// returns non-zero in case of error -typedef int (*rawFreeType)(intptr_t pool_id, void* raw_ptr, size_t raw_bytes); - -/* -MemPoolPolicy extension must be compatible with such structure fields layout - -struct MemPoolPolicy { - rawAllocType pAlloc; - rawFreeType pFree; - size_t granularity; // granularity of pAlloc allocations -}; -*/ - -struct MemPoolPolicy { - enum { - TBBMALLOC_POOL_VERSION = 1 - }; - - rawAllocType pAlloc; - rawFreeType pFree; - // granularity of pAlloc allocations. 0 means default used. - size_t granularity; - int version; - // all memory consumed at 1st pAlloc call and never returned, - // no more pAlloc calls after 1st - unsigned fixedPool : 1, - // memory consumed but returned only at pool termination - keepAllMemory : 1, - reserved : 30; - - MemPoolPolicy(rawAllocType pAlloc_, rawFreeType pFree_, - size_t granularity_ = 0, bool fixedPool_ = false, - bool keepAllMemory_ = false) : - pAlloc(pAlloc_), pFree(pFree_), granularity(granularity_), version(TBBMALLOC_POOL_VERSION), - fixedPool(fixedPool_), keepAllMemory(keepAllMemory_), - reserved(0) {} -}; - -// enums have same values as appropriate enums from ScalableAllocationResult -// TODO: use ScalableAllocationResult in pool_create directly -enum MemPoolError { - // pool created successfully - POOL_OK = TBBMALLOC_OK, - // invalid policy parameters found - INVALID_POLICY = TBBMALLOC_INVALID_PARAM, - // requested pool policy is not supported by allocator library - UNSUPPORTED_POLICY = TBBMALLOC_UNSUPPORTED, - // lack of memory during pool creation - NO_MEMORY = TBBMALLOC_NO_MEMORY, - // action takes no effect - NO_EFFECT = TBBMALLOC_NO_EFFECT -}; - -MemPoolError pool_create_v1(intptr_t pool_id, const MemPoolPolicy *policy, - rml::MemoryPool **pool); - -bool pool_destroy(MemoryPool* memPool); -void *pool_malloc(MemoryPool* memPool, size_t size); -void *pool_realloc(MemoryPool* memPool, void *object, size_t size); -void *pool_aligned_malloc(MemoryPool* mPool, size_t size, size_t alignment); -void *pool_aligned_realloc(MemoryPool* mPool, void *ptr, size_t size, size_t alignment); -bool pool_reset(MemoryPool* memPool); -bool pool_free(MemoryPool *memPool, void *object); -MemoryPool *pool_identify(void *object); -size_t pool_msize(MemoryPool *memPool, void *object); - -} // namespace rml - -#include /* To use new with the placement argument */ - -/* Ensure that including this header does not cause implicit linkage with TBB */ -#ifndef __TBB_NO_IMPLICIT_LINKAGE - #define __TBB_NO_IMPLICIT_LINKAGE 1 - #include "tbb_stddef.h" - #undef __TBB_NO_IMPLICIT_LINKAGE -#else - #include "tbb_stddef.h" -#endif - -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC -#include // std::forward -#endif - -#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT -#include -#endif - -namespace tbb { - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Workaround for erroneous "unreferenced parameter" warning in method destroy. - #pragma warning (push) - #pragma warning (disable: 4100) -#endif - -//! @cond INTERNAL -namespace internal { - -#if TBB_USE_EXCEPTIONS -// forward declaration is for inlining prevention -template __TBB_NOINLINE( void throw_exception(const E &e) ); -#endif - -// keep throw in a separate function to prevent code bloat -template -void throw_exception(const E &e) { - __TBB_THROW(e); -} - -} // namespace internal -//! @endcond - -//! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 -/** The members are ordered the same way they are in section 20.4.1 - of the ISO C++ standard. - @ingroup memory_allocation */ -template -class scalable_allocator { -public: - typedef typename internal::allocator_type::value_type value_type; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - template struct rebind { - typedef scalable_allocator other; - }; - - scalable_allocator() throw() {} - scalable_allocator( const scalable_allocator& ) throw() {} - template scalable_allocator(const scalable_allocator&) throw() {} - - pointer address(reference x) const {return &x;} - const_pointer address(const_reference x) const {return &x;} - - //! Allocate space for n objects. - pointer allocate( size_type n, const void* /*hint*/ =0 ) { - pointer p = static_cast( scalable_malloc( n * sizeof(value_type) ) ); - if (!p) - internal::throw_exception(std::bad_alloc()); - return p; - } - - //! Free previously allocated block of memory - void deallocate( pointer p, size_type ) { - scalable_free( p ); - } - - //! Largest value for which method allocate might succeed. - size_type max_size() const throw() { - size_type absolutemax = static_cast(-1) / sizeof (value_type); - return (absolutemax > 0 ? absolutemax : 1); - } -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - template - void construct(U *p, Args&&... args) - { ::new((void *)p) U(std::forward(args)...); } -#else /* __TBB_ALLOCATOR_CONSTRUCT_VARIADIC */ -#if __TBB_CPP11_RVALUE_REF_PRESENT - void construct( pointer p, value_type&& value ) { ::new((void*)(p)) value_type( std::move( value ) ); } -#endif - void construct( pointer p, const value_type& value ) {::new((void*)(p)) value_type(value);} -#endif /* __TBB_ALLOCATOR_CONSTRUCT_VARIADIC */ - void destroy( pointer p ) {p->~value_type();} -}; - -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif /* warning 4100 is back */ - -//! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1 -/** @ingroup memory_allocation */ -template<> -class scalable_allocator { -public: - typedef void* pointer; - typedef const void* const_pointer; - typedef void value_type; - template struct rebind { - typedef scalable_allocator other; - }; -}; - -template -inline bool operator==( const scalable_allocator&, const scalable_allocator& ) {return true;} - -template -inline bool operator!=( const scalable_allocator&, const scalable_allocator& ) {return false;} - -#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT - -namespace internal { - -//! C++17 memory resource implementation for scalable allocator -//! ISO C++ Section 23.12.2 -class scalable_resource_impl : public std::pmr::memory_resource { -private: - void* do_allocate(size_t bytes, size_t alignment) override { - void* ptr = scalable_aligned_malloc( bytes, alignment ); - if (!ptr) { - throw_exception(std::bad_alloc()); - } - return ptr; - } - - void do_deallocate(void* ptr, size_t /*bytes*/, size_t /*alignment*/) override { - scalable_free(ptr); - } - - //! Memory allocated by one instance of scalable_resource_impl could be deallocated by any - //! other instance of this class - bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override { - return this == &other || -#if __TBB_USE_OPTIONAL_RTTI - dynamic_cast(&other) != NULL; -#else - false; -#endif - } -}; - -} // namespace internal - -//! Global scalable allocator memory resource provider -inline std::pmr::memory_resource* scalable_memory_resource() noexcept { - static tbb::internal::scalable_resource_impl scalable_res; - return &scalable_res; -} - -#endif /* __TBB_CPP17_MEMORY_RESOURCE_PRESENT */ - -} // namespace tbb - -#if _MSC_VER - #if (__TBB_BUILD || __TBBMALLOC_BUILD) && !defined(__TBBMALLOC_NO_IMPLICIT_LINKAGE) - #define __TBBMALLOC_NO_IMPLICIT_LINKAGE 1 - #endif - - #if !__TBBMALLOC_NO_IMPLICIT_LINKAGE - #ifdef _DEBUG - #pragma comment(lib, "tbbmalloc_debug.lib") - #else - #pragma comment(lib, "tbbmalloc.lib") - #endif - #endif - - -#endif - -#endif /* __cplusplus */ - -#if !defined(__cplusplus) && __ICC==1100 - #pragma warning (pop) -#endif /* ICC 11.0 warning 991 is back */ - -#endif /* __TBB_scalable_allocator_H */ +#include "../oneapi/tbb/scalable_allocator.h" diff --git a/src/3rd party/tbb/spin_mutex.h b/src/3rd party/tbb/spin_mutex.h index 125849043a..1a6f7f077f 100644 --- a/src/3rd party/tbb/spin_mutex.h +++ b/src/3rd party/tbb/spin_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,201 +14,4 @@ limitations under the License. */ -#ifndef __TBB_spin_mutex_H -#define __TBB_spin_mutex_H - -#define __TBB_spin_mutex_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include -#include -#include "aligned_space.h" -#include "tbb_stddef.h" -#include "tbb_machine.h" -#include "tbb_profiling.h" -#include "internal/_mutex_padding.h" - -namespace tbb { - -//! A lock that occupies a single byte. -/** A spin_mutex is a spin mutex that fits in a single byte. - It should be used only for locking short critical sections - (typically less than 20 instructions) when fairness is not an issue. - If zero-initialized, the mutex is considered unheld. - @ingroup synchronization */ -class spin_mutex : internal::mutex_copy_deprecated_and_disabled { - //! 0 if lock is released, 1 if lock is acquired. - __TBB_atomic_flag flag; - -public: - //! Construct unacquired lock. - /** Equivalent to zero-initialization of *this. */ - spin_mutex() : flag(0) { -#if TBB_USE_THREADING_TOOLS - internal_construct(); -#endif - } - - //! Represents acquisition of a mutex. - class scoped_lock : internal::no_copy { - private: - //! Points to currently held mutex, or NULL if no lock is held. - spin_mutex* my_mutex; - - //! Value to store into spin_mutex::flag to unlock the mutex. - /** This variable is no longer used. Instead, 0 and 1 are used to - represent that the lock is free and acquired, respectively. - We keep the member variable here to ensure backward compatibility */ - __TBB_Flag my_unlock_value; - - //! Like acquire, but with ITT instrumentation. - void __TBB_EXPORTED_METHOD internal_acquire( spin_mutex& m ); - - //! Like try_acquire, but with ITT instrumentation. - bool __TBB_EXPORTED_METHOD internal_try_acquire( spin_mutex& m ); - - //! Like release, but with ITT instrumentation. - void __TBB_EXPORTED_METHOD internal_release(); - - friend class spin_mutex; - - public: - //! Construct without acquiring a mutex. - scoped_lock() : my_mutex(NULL), my_unlock_value(0) {} - - //! Construct and acquire lock on a mutex. - scoped_lock( spin_mutex& m ) : my_unlock_value(0) { - internal::suppress_unused_warning(my_unlock_value); -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - my_mutex=NULL; - internal_acquire(m); -#else - my_mutex=&m; - __TBB_LockByte(m.flag); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT*/ - } - - //! Acquire lock. - void acquire( spin_mutex& m ) { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - internal_acquire(m); -#else - my_mutex = &m; - __TBB_LockByte(m.flag); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT*/ - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_acquire( spin_mutex& m ) { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - return internal_try_acquire(m); -#else - bool result = __TBB_TryLockByte(m.flag); - if( result ) - my_mutex = &m; - return result; -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT*/ - } - - //! Release lock - void release() { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - internal_release(); -#else - __TBB_UnlockByte(my_mutex->flag); - my_mutex = NULL; -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - - //! Destroy lock. If holding a lock, releases the lock first. - ~scoped_lock() { - if( my_mutex ) { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - internal_release(); -#else - __TBB_UnlockByte(my_mutex->flag); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - } - }; - - //! Internal constructor with ITT instrumentation. - void __TBB_EXPORTED_METHOD internal_construct(); - - // Mutex traits - static const bool is_rw_mutex = false; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = false; - - // ISO C++0x compatibility methods - - //! Acquire lock - void lock() { -#if TBB_USE_THREADING_TOOLS - aligned_space tmp; - new(tmp.begin()) scoped_lock(*this); -#else - __TBB_LockByte(flag); -#endif /* TBB_USE_THREADING_TOOLS*/ - } - - //! Try acquiring lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_lock() { -#if TBB_USE_THREADING_TOOLS - aligned_space tmp; - return (new(tmp.begin()) scoped_lock)->internal_try_acquire(*this); -#else - return __TBB_TryLockByte(flag); -#endif /* TBB_USE_THREADING_TOOLS*/ - } - - //! Release lock - void unlock() { -#if TBB_USE_THREADING_TOOLS - aligned_space tmp; - scoped_lock& s = *tmp.begin(); - s.my_mutex = this; - s.internal_release(); -#else - __TBB_UnlockByte(flag); -#endif /* TBB_USE_THREADING_TOOLS */ - } - - friend class scoped_lock; -}; // end of spin_mutex - -__TBB_DEFINE_PROFILING_SET_NAME(spin_mutex) - -} // namespace tbb - -#if ( __TBB_x86_32 || __TBB_x86_64 ) -#include "internal/_x86_eliding_mutex_impl.h" -#endif - -namespace tbb { -//! A cross-platform spin mutex with speculative lock acquisition. -/** On platforms with proper HW support, this lock may speculatively execute - its critical sections, using HW mechanisms to detect real data races and - ensure atomicity of the critical sections. In particular, it uses - Intel(R) Transactional Synchronization Extensions (Intel(R) TSX). - Without such HW support, it behaves like a spin_mutex. - It should be used for locking short critical sections where the lock is - contended but the data it protects are not. If zero-initialized, the - mutex is considered unheld. - @ingroup synchronization */ - -#if ( __TBB_x86_32 || __TBB_x86_64 ) -typedef interface7::internal::padded_mutex speculative_spin_mutex; -#else -typedef interface7::internal::padded_mutex speculative_spin_mutex; -#endif -__TBB_DEFINE_PROFILING_SET_NAME(speculative_spin_mutex) - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_spin_mutex_H_include_area - -#endif /* __TBB_spin_mutex_H */ +#include "../oneapi/tbb/spin_mutex.h" diff --git a/src/3rd party/tbb/spin_rw_mutex.h b/src/3rd party/tbb/spin_rw_mutex.h index 1477e684fd..d36282b486 100644 --- a/src/3rd party/tbb/spin_rw_mutex.h +++ b/src/3rd party/tbb/spin_rw_mutex.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,239 +14,4 @@ limitations under the License. */ -#ifndef __TBB_spin_rw_mutex_H -#define __TBB_spin_rw_mutex_H - -#include "tbb_stddef.h" -#include "tbb_machine.h" -#include "tbb_profiling.h" -#include "internal/_mutex_padding.h" - -namespace tbb { - -#if __TBB_TSX_AVAILABLE -namespace interface8 { namespace internal { - class x86_rtm_rw_mutex; -}} -#endif - -class spin_rw_mutex_v3; -typedef spin_rw_mutex_v3 spin_rw_mutex; - -//! Fast, unfair, spinning reader-writer lock with backoff and writer-preference -/** @ingroup synchronization */ -class spin_rw_mutex_v3 : internal::mutex_copy_deprecated_and_disabled { - //! @cond INTERNAL - - //! Internal acquire write lock. - bool __TBB_EXPORTED_METHOD internal_acquire_writer(); - - //! Out of line code for releasing a write lock. - /** This code has debug checking and instrumentation for Intel(R) Thread Checker and Intel(R) Thread Profiler. */ - void __TBB_EXPORTED_METHOD internal_release_writer(); - - //! Internal acquire read lock. - void __TBB_EXPORTED_METHOD internal_acquire_reader(); - - //! Internal upgrade reader to become a writer. - bool __TBB_EXPORTED_METHOD internal_upgrade(); - - //! Out of line code for downgrading a writer to a reader. - /** This code has debug checking and instrumentation for Intel(R) Thread Checker and Intel(R) Thread Profiler. */ - void __TBB_EXPORTED_METHOD internal_downgrade(); - - //! Internal release read lock. - void __TBB_EXPORTED_METHOD internal_release_reader(); - - //! Internal try_acquire write lock. - bool __TBB_EXPORTED_METHOD internal_try_acquire_writer(); - - //! Internal try_acquire read lock. - bool __TBB_EXPORTED_METHOD internal_try_acquire_reader(); - - //! @endcond -public: - //! Construct unacquired mutex. - spin_rw_mutex_v3() : state(0) { -#if TBB_USE_THREADING_TOOLS - internal_construct(); -#endif - } - -#if TBB_USE_ASSERT - //! Destructor asserts if the mutex is acquired, i.e. state is zero. - ~spin_rw_mutex_v3() { - __TBB_ASSERT( !state, "destruction of an acquired mutex"); - }; -#endif /* TBB_USE_ASSERT */ - - //! The scoped locking pattern - /** It helps to avoid the common problem of forgetting to release lock. - It also nicely provides the "node" for queuing locks. */ - class scoped_lock : internal::no_copy { -#if __TBB_TSX_AVAILABLE - friend class tbb::interface8::internal::x86_rtm_rw_mutex; -#endif - public: - //! Construct lock that has not acquired a mutex. - /** Equivalent to zero-initialization of *this. */ - scoped_lock() : mutex(NULL), is_writer(false) {} - - //! Acquire lock on given mutex. - scoped_lock( spin_rw_mutex& m, bool write = true ) : mutex(NULL) { - acquire(m, write); - } - - //! Release lock (if lock is held). - ~scoped_lock() { - if( mutex ) release(); - } - - //! Acquire lock on given mutex. - void acquire( spin_rw_mutex& m, bool write = true ) { - __TBB_ASSERT( !mutex, "holding mutex already" ); - is_writer = write; - mutex = &m; - if( write ) mutex->internal_acquire_writer(); - else mutex->internal_acquire_reader(); - } - - //! Upgrade reader to become a writer. - /** Returns whether the upgrade happened without releasing and re-acquiring the lock */ - bool upgrade_to_writer() { - __TBB_ASSERT( mutex, "mutex is not acquired" ); - if (is_writer) return true; // Already a writer - is_writer = true; - return mutex->internal_upgrade(); - } - - //! Release lock. - void release() { - __TBB_ASSERT( mutex, "mutex is not acquired" ); - spin_rw_mutex *m = mutex; - mutex = NULL; -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - if( is_writer ) m->internal_release_writer(); - else m->internal_release_reader(); -#else - if( is_writer ) __TBB_AtomicAND( &m->state, READERS ); - else __TBB_FetchAndAddWrelease( &m->state, -(intptr_t)ONE_READER); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - - //! Downgrade writer to become a reader. - bool downgrade_to_reader() { - __TBB_ASSERT( mutex, "mutex is not acquired" ); - if (!is_writer) return true; // Already a reader -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - mutex->internal_downgrade(); -#else - __TBB_FetchAndAddW( &mutex->state, ((intptr_t)ONE_READER-WRITER)); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - is_writer = false; - return true; - } - - //! Try acquire lock on given mutex. - bool try_acquire( spin_rw_mutex& m, bool write = true ) { - __TBB_ASSERT( !mutex, "holding mutex already" ); - bool result; - is_writer = write; - result = write? m.internal_try_acquire_writer() - : m.internal_try_acquire_reader(); - if( result ) - mutex = &m; - return result; - } - - protected: - - //! The pointer to the current mutex that is held, or NULL if no mutex is held. - spin_rw_mutex* mutex; - - //! If mutex!=NULL, then is_writer is true if holding a writer lock, false if holding a reader lock. - /** Not defined if not holding a lock. */ - bool is_writer; - }; - - // Mutex traits - static const bool is_rw_mutex = true; - static const bool is_recursive_mutex = false; - static const bool is_fair_mutex = false; - - // ISO C++0x compatibility methods - - //! Acquire writer lock - void lock() {internal_acquire_writer();} - - //! Try acquiring writer lock (non-blocking) - /** Return true if lock acquired; false otherwise. */ - bool try_lock() {return internal_try_acquire_writer();} - - //! Release lock - void unlock() { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - if( state&WRITER ) internal_release_writer(); - else internal_release_reader(); -#else - if( state&WRITER ) __TBB_AtomicAND( &state, READERS ); - else __TBB_FetchAndAddWrelease( &state, -(intptr_t)ONE_READER); -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - - // Methods for reader locks that resemble ISO C++0x compatibility methods. - - //! Acquire reader lock - void lock_read() {internal_acquire_reader();} - - //! Try acquiring reader lock (non-blocking) - /** Return true if reader lock acquired; false otherwise. */ - bool try_lock_read() {return internal_try_acquire_reader();} - -protected: - typedef intptr_t state_t; - static const state_t WRITER = 1; - static const state_t WRITER_PENDING = 2; - static const state_t READERS = ~(WRITER | WRITER_PENDING); - static const state_t ONE_READER = 4; - static const state_t BUSY = WRITER | READERS; - //! State of lock - /** Bit 0 = writer is holding lock - Bit 1 = request by a writer to acquire lock (hint to readers to wait) - Bit 2..N = number of readers holding lock */ - state_t state; - -private: - void __TBB_EXPORTED_METHOD internal_construct(); -}; - -__TBB_DEFINE_PROFILING_SET_NAME(spin_rw_mutex) - -} // namespace tbb - -#if __TBB_TSX_AVAILABLE -#include "internal/_x86_rtm_rw_mutex_impl.h" -#endif - -namespace tbb { -namespace interface8 { -//! A cross-platform spin reader/writer mutex with speculative lock acquisition. -/** On platforms with proper HW support, this lock may speculatively execute - its critical sections, using HW mechanisms to detect real data races and - ensure atomicity of the critical sections. In particular, it uses - Intel(R) Transactional Synchronization Extensions (Intel(R) TSX). - Without such HW support, it behaves like a spin_rw_mutex. - It should be used for locking short critical sections where the lock is - contended but the data it protects are not. - @ingroup synchronization */ -#if __TBB_TSX_AVAILABLE -typedef interface7::internal::padded_mutex speculative_spin_rw_mutex; -#else -typedef interface7::internal::padded_mutex speculative_spin_rw_mutex; -#endif -} // namespace interface8 - -using interface8::speculative_spin_rw_mutex; -__TBB_DEFINE_PROFILING_SET_NAME(speculative_spin_rw_mutex) -} // namespace tbb -#endif /* __TBB_spin_rw_mutex_H */ +#include "../oneapi/tbb/spin_rw_mutex.h" diff --git a/src/3rd party/tbb/task.h b/src/3rd party/tbb/task.h index 15cb88c877..9be95b0d69 100644 --- a/src/3rd party/tbb/task.h +++ b/src/3rd party/tbb/task.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,1165 +14,4 @@ limitations under the License. */ -#ifndef __TBB_task_H -#define __TBB_task_H - -#define __TBB_task_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "tbb_stddef.h" -#include "tbb_machine.h" -#include "tbb_profiling.h" -#include - -typedef struct ___itt_caller *__itt_caller; - -namespace tbb { - -class task; -class task_list; -class task_group_context; - -// MSVC does not allow taking the address of a member that was defined -// privately in task_base and made public in class task via a using declaration. -#if _MSC_VER || (__GNUC__==3 && __GNUC_MINOR__<3) -#define __TBB_TASK_BASE_ACCESS public -#else -#define __TBB_TASK_BASE_ACCESS private -#endif - -namespace internal { //< @cond INTERNAL - - class allocate_additional_child_of_proxy: no_assign { - //! No longer used, but retained for binary layout compatibility. Always NULL. - task* self; - task& parent; - public: - explicit allocate_additional_child_of_proxy( task& parent_ ) : self(NULL), parent(parent_) { - suppress_unused_warning( self ); - } - task& __TBB_EXPORTED_METHOD allocate( size_t size ) const; - void __TBB_EXPORTED_METHOD free( task& ) const; - }; - - struct cpu_ctl_env_space { int space[sizeof(internal::uint64_t)/sizeof(int)]; }; -} //< namespace internal @endcond - -namespace interface5 { - namespace internal { - //! Base class for methods that became static in TBB 3.0. - /** TBB's evolution caused the "this" argument for several methods to become obsolete. - However, for backwards binary compatibility, the new methods need distinct names, - otherwise the One Definition Rule would be broken. Hence the new methods are - defined in this private base class, and then exposed in class task via - using declarations. */ - class task_base: tbb::internal::no_copy { - __TBB_TASK_BASE_ACCESS: - friend class tbb::task; - - //! Schedule task for execution when a worker becomes available. - static void spawn( task& t ); - - //! Spawn multiple tasks and clear list. - static void spawn( task_list& list ); - - //! Like allocate_child, except that task's parent becomes "t", not this. - /** Typically used in conjunction with schedule_to_reexecute to implement while loops. - Atomically increments the reference count of t.parent() */ - static tbb::internal::allocate_additional_child_of_proxy allocate_additional_child_of( task& t ) { - return tbb::internal::allocate_additional_child_of_proxy(t); - } - - //! Destroy a task. - /** Usually, calling this method is unnecessary, because a task is - implicitly deleted after its execute() method runs. However, - sometimes a task needs to be explicitly deallocated, such as - when a root task is used as the parent in spawn_and_wait_for_all. */ - static void __TBB_EXPORTED_FUNC destroy( task& victim ); - }; - } // internal -} // interface5 - -//! @cond INTERNAL -namespace internal { - - class scheduler: no_copy { - public: - //! For internal use only - virtual void spawn( task& first, task*& next ) = 0; - - //! For internal use only - virtual void wait_for_all( task& parent, task* child ) = 0; - - //! For internal use only - virtual void spawn_root_and_wait( task& first, task*& next ) = 0; - - //! Pure virtual destructor; - // Have to have it just to shut up overzealous compilation warnings - virtual ~scheduler() = 0; - - //! For internal use only - virtual void enqueue( task& t, void* reserved ) = 0; - }; - - //! A reference count - /** Should always be non-negative. A signed type is used so that underflow can be detected. */ - typedef intptr_t reference_count; - -#if __TBB_PREVIEW_RESUMABLE_TASKS - //! The flag to indicate that the wait task has been abandoned. - static const reference_count abandon_flag = reference_count(1) << (sizeof(reference_count)*CHAR_BIT - 2); -#endif - - //! An id as used for specifying affinity. - typedef unsigned short affinity_id; - -#if __TBB_TASK_ISOLATION - //! A tag for task isolation. - typedef intptr_t isolation_tag; - const isolation_tag no_isolation = 0; -#endif /* __TBB_TASK_ISOLATION */ - -#if __TBB_TASK_GROUP_CONTEXT - class generic_scheduler; - - struct context_list_node_t { - context_list_node_t *my_prev, - *my_next; - }; - - class allocate_root_with_context_proxy: no_assign { - task_group_context& my_context; - public: - allocate_root_with_context_proxy ( task_group_context& ctx ) : my_context(ctx) {} - task& __TBB_EXPORTED_METHOD allocate( size_t size ) const; - void __TBB_EXPORTED_METHOD free( task& ) const; - }; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - - class allocate_root_proxy: no_assign { - public: - static task& __TBB_EXPORTED_FUNC allocate( size_t size ); - static void __TBB_EXPORTED_FUNC free( task& ); - }; - - class allocate_continuation_proxy: no_assign { - public: - task& __TBB_EXPORTED_METHOD allocate( size_t size ) const; - void __TBB_EXPORTED_METHOD free( task& ) const; - }; - - class allocate_child_proxy: no_assign { - public: - task& __TBB_EXPORTED_METHOD allocate( size_t size ) const; - void __TBB_EXPORTED_METHOD free( task& ) const; - }; - -#if __TBB_PREVIEW_CRITICAL_TASKS - // TODO: move to class methods when critical task API becomes public - void make_critical( task& t ); - bool is_critical( task& t ); -#endif - - //! Memory prefix to a task object. - /** This class is internal to the library. - Do not reference it directly, except within the library itself. - Fields are ordered in way that preserves backwards compatibility and yields good packing on - typical 32-bit and 64-bit platforms. New fields should be added at the beginning for - backward compatibility with accesses to the task prefix inlined into application code. To - prevent ODR violation, the class shall have the same layout in all application translation - units. If some fields are conditional (e.g. enabled by preview macros) and might get - skipped, use reserved fields to adjust the layout. - - In case task prefix size exceeds 32 or 64 bytes on IA32 and Intel64 architectures - correspondingly, consider dynamic setting of task_alignment and task_prefix_reservation_size - based on the maximal operand size supported by the current CPU. - - @ingroup task_scheduling */ - class task_prefix { - private: - friend class tbb::task; - friend class tbb::interface5::internal::task_base; - friend class tbb::task_list; - friend class internal::scheduler; - friend class internal::allocate_root_proxy; - friend class internal::allocate_child_proxy; - friend class internal::allocate_continuation_proxy; - friend class internal::allocate_additional_child_of_proxy; -#if __TBB_PREVIEW_CRITICAL_TASKS - friend void make_critical( task& ); - friend bool is_critical( task& ); -#endif - -#if __TBB_TASK_ISOLATION - //! The tag used for task isolation. - isolation_tag isolation; -#else - intptr_t reserved_space_for_task_isolation_tag; -#endif /* __TBB_TASK_ISOLATION */ - -#if __TBB_TASK_GROUP_CONTEXT - //! Shared context that is used to communicate asynchronous state changes - /** Currently it is used to broadcast cancellation requests generated both - by users and as the result of unhandled exceptions in the task::execute() - methods. */ - task_group_context *context; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - - //! The scheduler that allocated the task, or NULL if the task is big. - /** Small tasks are pooled by the scheduler that allocated the task. - If a scheduler needs to free a small task allocated by another scheduler, - it returns the task to that other scheduler. This policy avoids - memory space blowup issues for memory allocators that allocate from - thread-specific pools. */ - scheduler* origin; - -#if __TBB_TASK_PRIORITY || __TBB_PREVIEW_RESUMABLE_TASKS - union { -#endif /* __TBB_TASK_PRIORITY */ - //! Obsolete. The scheduler that owns the task. - /** Retained only for the sake of backward binary compatibility. - Still used by inline methods in the task.h header. **/ - scheduler* owner; - -#if __TBB_TASK_PRIORITY - //! Pointer to the next offloaded lower priority task. - /** Used to maintain a list of offloaded tasks inside the scheduler. **/ - task* next_offloaded; -#endif - -#if __TBB_PREVIEW_RESUMABLE_TASKS - //! Pointer to the abandoned scheduler where the current task is waited for. - scheduler* abandoned_scheduler; -#endif -#if __TBB_TASK_PRIORITY || __TBB_PREVIEW_RESUMABLE_TASKS - }; -#endif /* __TBB_TASK_PRIORITY || __TBB_PREVIEW_RESUMABLE_TASKS */ - - //! The task whose reference count includes me. - /** In the "blocking style" of programming, this field points to the parent task. - In the "continuation-passing style" of programming, this field points to the - continuation of the parent. */ - tbb::task* parent; - - //! Reference count used for synchronization. - /** In the "continuation-passing style" of programming, this field is - the difference of the number of allocated children minus the - number of children that have completed. - In the "blocking style" of programming, this field is one more than the difference. */ - __TBB_atomic reference_count ref_count; - - //! Obsolete. Used to be scheduling depth before TBB 2.2 - /** Retained only for the sake of backward binary compatibility. - Not used by TBB anymore. **/ - int depth; - - //! A task::state_type, stored as a byte for compactness. - /** This state is exposed to users via method task::state(). */ - unsigned char state; - - //! Miscellaneous state that is not directly visible to users, stored as a byte for compactness. - /** 0x0 -> version 1.0 task - 0x1 -> version >=2.1 task - 0x10 -> task was enqueued - 0x20 -> task_proxy - 0x40 -> task has live ref_count - 0x80 -> a stolen task */ - unsigned char extra_state; - - affinity_id affinity; - - //! "next" field for list of task - tbb::task* next; - - //! The task corresponding to this task_prefix. - tbb::task& task() {return *reinterpret_cast(this+1);} - }; - -} // namespace internal -//! @endcond - -#if __TBB_TASK_GROUP_CONTEXT - -#if __TBB_TASK_PRIORITY -namespace internal { - static const int priority_stride_v4 = INT_MAX / 4; -#if __TBB_PREVIEW_CRITICAL_TASKS - // TODO: move into priority_t enum when critical tasks become public feature - static const int priority_critical = priority_stride_v4 * 3 + priority_stride_v4 / 3 * 2; -#endif -} - -enum priority_t { - priority_normal = internal::priority_stride_v4 * 2, - priority_low = priority_normal - internal::priority_stride_v4, - priority_high = priority_normal + internal::priority_stride_v4 -}; - -#endif /* __TBB_TASK_PRIORITY */ - -#if TBB_USE_CAPTURED_EXCEPTION - class tbb_exception; -#else - namespace internal { - class tbb_exception_ptr; - } -#endif /* !TBB_USE_CAPTURED_EXCEPTION */ - -class task_scheduler_init; -namespace interface7 { class task_arena; } -using interface7::task_arena; - -//! Used to form groups of tasks -/** @ingroup task_scheduling - The context services explicit cancellation requests from user code, and unhandled - exceptions intercepted during tasks execution. Intercepting an exception results - in generating internal cancellation requests (which is processed in exactly the - same way as external ones). - - The context is associated with one or more root tasks and defines the cancellation - group that includes all the descendants of the corresponding root task(s). Association - is established when a context object is passed as an argument to the task::allocate_root() - method. See task_group_context::task_group_context for more details. - - The context can be bound to another one, and other contexts can be bound to it, - forming a tree-like structure: parent -> this -> children. Arrows here designate - cancellation propagation direction. If a task in a cancellation group is cancelled - all the other tasks in this group and groups bound to it (as children) get cancelled too. - - IMPLEMENTATION NOTE: - When adding new members to task_group_context or changing types of existing ones, - update the size of both padding buffers (_leading_padding and _trailing_padding) - appropriately. See also VERSIONING NOTE at the constructor definition below. **/ -class task_group_context : internal::no_copy { -private: - friend class internal::generic_scheduler; - friend class task_scheduler_init; - friend class task_arena; - -#if TBB_USE_CAPTURED_EXCEPTION - typedef tbb_exception exception_container_type; -#else - typedef internal::tbb_exception_ptr exception_container_type; -#endif - - enum version_traits_word_layout { - traits_offset = 16, - version_mask = 0xFFFF, - traits_mask = 0xFFFFul << traits_offset - }; - -public: - enum kind_type { - isolated, - bound - }; - - enum traits_type { - exact_exception = 0x0001ul << traits_offset, -#if __TBB_FP_CONTEXT - fp_settings = 0x0002ul << traits_offset, -#endif - concurrent_wait = 0x0004ul << traits_offset, -#if TBB_USE_CAPTURED_EXCEPTION - default_traits = 0 -#else - default_traits = exact_exception -#endif /* !TBB_USE_CAPTURED_EXCEPTION */ - }; - -private: - enum state { - may_have_children = 1, - // the following enumerations must be the last, new 2^x values must go above - next_state_value, low_unused_state_bit = (next_state_value-1)*2 - }; - - union { - //! Flavor of this context: bound or isolated. - // TODO: describe asynchronous use, and whether any memory semantics are needed - __TBB_atomic kind_type my_kind; - uintptr_t _my_kind_aligner; - }; - - //! Pointer to the context of the parent cancellation group. NULL for isolated contexts. - task_group_context *my_parent; - - //! Used to form the thread specific list of contexts without additional memory allocation. - /** A context is included into the list of the current thread when its binding to - its parent happens. Any context can be present in the list of one thread only. **/ - internal::context_list_node_t my_node; - - //! Used to set and maintain stack stitching point for Intel Performance Tools. - __itt_caller itt_caller; - - //! Leading padding protecting accesses to frequently used members from false sharing. - /** Read accesses to the field my_cancellation_requested are on the hot path inside - the scheduler. This padding ensures that this field never shares the same cache - line with a local variable that is frequently written to. **/ - char _leading_padding[internal::NFS_MaxLineSize - - 2 * sizeof(uintptr_t)- sizeof(void*) - sizeof(internal::context_list_node_t) - - sizeof(__itt_caller) -#if __TBB_FP_CONTEXT - - sizeof(internal::cpu_ctl_env_space) -#endif - ]; - -#if __TBB_FP_CONTEXT - //! Space for platform-specific FPU settings. - /** Must only be accessed inside TBB binaries, and never directly in user - code or inline methods. */ - internal::cpu_ctl_env_space my_cpu_ctl_env; -#endif - - //! Specifies whether cancellation was requested for this task group. - uintptr_t my_cancellation_requested; - - //! Version for run-time checks and behavioral traits of the context. - /** Version occupies low 16 bits, and traits (zero or more ORed enumerators - from the traits_type enumerations) take the next 16 bits. - Original (zeroth) version of the context did not support any traits. **/ - uintptr_t my_version_and_traits; - - //! Pointer to the container storing exception being propagated across this task group. - exception_container_type *my_exception; - - //! Scheduler instance that registered this context in its thread specific list. - internal::generic_scheduler *my_owner; - - //! Internal state (combination of state flags, currently only may_have_children). - uintptr_t my_state; - -#if __TBB_TASK_PRIORITY - //! Priority level of the task group (in normalized representation) - intptr_t my_priority; -#endif /* __TBB_TASK_PRIORITY */ - - //! Description of algorithm for scheduler based instrumentation. - internal::string_index my_name; - - //! Trailing padding protecting accesses to frequently used members from false sharing - /** \sa _leading_padding **/ - char _trailing_padding[internal::NFS_MaxLineSize - 2 * sizeof(uintptr_t) - 2 * sizeof(void*) -#if __TBB_TASK_PRIORITY - - sizeof(intptr_t) -#endif /* __TBB_TASK_PRIORITY */ - - sizeof(internal::string_index) - ]; - -public: - //! Default & binding constructor. - /** By default a bound context is created. That is this context will be bound - (as child) to the context of the task calling task::allocate_root(this_context) - method. Cancellation requests passed to the parent context are propagated - to all the contexts bound to it. Similarly priority change is propagated - from the parent context to its children. - - If task_group_context::isolated is used as the argument, then the tasks associated - with this context will never be affected by events in any other context. - - Creating isolated contexts involve much less overhead, but they have limited - utility. Normally when an exception occurs in an algorithm that has nested - ones running, it is desirably to have all the nested algorithms cancelled - as well. Such a behavior requires nested algorithms to use bound contexts. - - There is one good place where using isolated algorithms is beneficial. It is - a master thread. That is if a particular algorithm is invoked directly from - the master thread (not from a TBB task), supplying it with explicitly - created isolated context will result in a faster algorithm startup. - - VERSIONING NOTE: - Implementation(s) of task_group_context constructor(s) cannot be made - entirely out-of-line because the run-time version must be set by the user - code. This will become critically important for binary compatibility, if - we ever have to change the size of the context object. - - Boosting the runtime version will also be necessary if new data fields are - introduced in the currently unused padding areas and these fields are updated - by inline methods. **/ - task_group_context ( kind_type relation_with_parent = bound, - uintptr_t t = default_traits ) - : my_kind(relation_with_parent) - , my_version_and_traits(3 | t) - , my_name(internal::CUSTOM_CTX) - { - init(); - } - - // Custom constructor for instrumentation of tbb algorithm - task_group_context ( internal::string_index name ) - : my_kind(bound) - , my_version_and_traits(3 | default_traits) - , my_name(name) - { - init(); - } - - // Do not introduce standalone unbind method since it will break state propagation assumptions - __TBB_EXPORTED_METHOD ~task_group_context (); - - //! Forcefully reinitializes the context after the task tree it was associated with is completed. - /** Because the method assumes that all the tasks that used to be associated with - this context have already finished, calling it while the context is still - in use somewhere in the task hierarchy leads to undefined behavior. - - IMPORTANT: This method is not thread safe! - - The method does not change the context's parent if it is set. **/ - void __TBB_EXPORTED_METHOD reset (); - - //! Initiates cancellation of all tasks in this cancellation group and its subordinate groups. - /** \return false if cancellation has already been requested, true otherwise. - - Note that canceling never fails. When false is returned, it just means that - another thread (or this one) has already sent cancellation request to this - context or to one of its ancestors (if this context is bound). It is guaranteed - that when this method is concurrently called on the same not yet cancelled - context, true will be returned by one and only one invocation. **/ - bool __TBB_EXPORTED_METHOD cancel_group_execution (); - - //! Returns true if the context received cancellation request. - bool __TBB_EXPORTED_METHOD is_group_execution_cancelled () const; - - //! Records the pending exception, and cancels the task group. - /** May be called only from inside a catch-block. If the context is already - cancelled, does nothing. - The method brings the task group associated with this context exactly into - the state it would be in, if one of its tasks threw the currently pending - exception during its execution. In other words, it emulates the actions - of the scheduler's dispatch loop exception handler. **/ - void __TBB_EXPORTED_METHOD register_pending_exception (); - -#if __TBB_FP_CONTEXT - //! Captures the current FPU control settings to the context. - /** Because the method assumes that all the tasks that used to be associated with - this context have already finished, calling it while the context is still - in use somewhere in the task hierarchy leads to undefined behavior. - - IMPORTANT: This method is not thread safe! - - The method does not change the FPU control settings of the context's parent. **/ - void __TBB_EXPORTED_METHOD capture_fp_settings (); -#endif - -#if __TBB_TASK_PRIORITY - //! Changes priority of the task group - __TBB_DEPRECATED void set_priority ( priority_t ); - - //! Retrieves current priority of the current task group - __TBB_DEPRECATED priority_t priority () const; -#endif /* __TBB_TASK_PRIORITY */ - - //! Returns the context's trait - uintptr_t traits() const { return my_version_and_traits & traits_mask; } - -protected: - //! Out-of-line part of the constructor. - /** Singled out to ensure backward binary compatibility of the future versions. **/ - void __TBB_EXPORTED_METHOD init (); - -private: - friend class task; - friend class internal::allocate_root_with_context_proxy; - - static const kind_type binding_required = bound; - static const kind_type binding_completed = kind_type(bound+1); - static const kind_type detached = kind_type(binding_completed+1); - static const kind_type dying = kind_type(detached+1); - - //! Propagates any state change detected to *this, and as an optimisation possibly also upward along the heritage line. - template - void propagate_task_group_state ( T task_group_context::*mptr_state, task_group_context& src, T new_state ); - - //! Registers this context with the local scheduler and binds it to its parent context - void bind_to ( internal::generic_scheduler *local_sched ); - - //! Registers this context with the local scheduler - void register_with ( internal::generic_scheduler *local_sched ); - -#if __TBB_FP_CONTEXT - //! Copies FPU control setting from another context - // TODO: Consider adding #else stub in order to omit #if sections in other code - void copy_fp_settings( const task_group_context &src ); -#endif /* __TBB_FP_CONTEXT */ -}; // class task_group_context - -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -//! Base class for user-defined tasks. -/** @ingroup task_scheduling */ -class task: __TBB_TASK_BASE_ACCESS interface5::internal::task_base { - - //! Set reference count - void __TBB_EXPORTED_METHOD internal_set_ref_count( int count ); - - //! Decrement reference count and return its new value. - internal::reference_count __TBB_EXPORTED_METHOD internal_decrement_ref_count(); - -protected: - //! Default constructor. - task() {prefix().extra_state=1;} - -public: - //! Destructor. - virtual ~task() {} - - //! Should be overridden by derived classes. - virtual task* execute() = 0; - - //! Enumeration of task states that the scheduler considers. - enum state_type { - //! task is running, and will be destroyed after method execute() completes. - executing, - //! task to be rescheduled. - reexecute, - //! task is in ready pool, or is going to be put there, or was just taken off. - ready, - //! task object is freshly allocated or recycled. - allocated, - //! task object is on free list, or is going to be put there, or was just taken off. - freed, - //! task to be recycled as continuation - recycle -#if __TBB_RECYCLE_TO_ENQUEUE - //! task to be scheduled for starvation-resistant execution - ,to_enqueue -#endif -#if __TBB_PREVIEW_RESUMABLE_TASKS - //! a special task used to resume a scheduler. - ,to_resume -#endif - }; - - //------------------------------------------------------------------------ - // Allocating tasks - //------------------------------------------------------------------------ - - //! Returns proxy for overloaded new that allocates a root task. - static internal::allocate_root_proxy allocate_root() { - return internal::allocate_root_proxy(); - } - -#if __TBB_TASK_GROUP_CONTEXT - //! Returns proxy for overloaded new that allocates a root task associated with user supplied context. - static internal::allocate_root_with_context_proxy allocate_root( task_group_context& ctx ) { - return internal::allocate_root_with_context_proxy(ctx); - } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - - //! Returns proxy for overloaded new that allocates a continuation task of *this. - /** The continuation's parent becomes the parent of *this. */ - internal::allocate_continuation_proxy& allocate_continuation() { - return *reinterpret_cast(this); - } - - //! Returns proxy for overloaded new that allocates a child task of *this. - internal::allocate_child_proxy& allocate_child() { - return *reinterpret_cast(this); - } - - //! Define recommended static form via import from base class. - using task_base::allocate_additional_child_of; - -#if __TBB_DEPRECATED_TASK_INTERFACE - //! Destroy a task. - /** Usually, calling this method is unnecessary, because a task is - implicitly deleted after its execute() method runs. However, - sometimes a task needs to be explicitly deallocated, such as - when a root task is used as the parent in spawn_and_wait_for_all. */ - void __TBB_EXPORTED_METHOD destroy( task& t ); -#else /* !__TBB_DEPRECATED_TASK_INTERFACE */ - //! Define recommended static form via import from base class. - using task_base::destroy; -#endif /* !__TBB_DEPRECATED_TASK_INTERFACE */ - - //------------------------------------------------------------------------ - // Recycling of tasks - //------------------------------------------------------------------------ - - //! Change this to be a continuation of its former self. - /** The caller must guarantee that the task's refcount does not become zero until - after the method execute() returns. Typically, this is done by having - method execute() return a pointer to a child of the task. If the guarantee - cannot be made, use method recycle_as_safe_continuation instead. - - Because of the hazard, this method may be deprecated in the future. */ - void recycle_as_continuation() { - __TBB_ASSERT( prefix().state==executing, "execute not running?" ); - prefix().state = allocated; - } - - //! Recommended to use, safe variant of recycle_as_continuation - /** For safety, it requires additional increment of ref_count. - With no descendants and ref_count of 1, it has the semantics of recycle_to_reexecute. */ - void recycle_as_safe_continuation() { - __TBB_ASSERT( prefix().state==executing, "execute not running?" ); - prefix().state = recycle; - } - - //! Change this to be a child of new_parent. - void recycle_as_child_of( task& new_parent ) { - internal::task_prefix& p = prefix(); - __TBB_ASSERT( prefix().state==executing||prefix().state==allocated, "execute not running, or already recycled" ); - __TBB_ASSERT( prefix().ref_count==0, "no child tasks allowed when recycled as a child" ); - __TBB_ASSERT( p.parent==NULL, "parent must be null" ); - __TBB_ASSERT( new_parent.prefix().state<=recycle, "corrupt parent's state" ); - __TBB_ASSERT( new_parent.prefix().state!=freed, "parent already freed" ); - p.state = allocated; - p.parent = &new_parent; -#if __TBB_TASK_GROUP_CONTEXT - p.context = new_parent.prefix().context; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - } - - //! Schedule this for reexecution after current execute() returns. - /** Made obsolete by recycle_as_safe_continuation; may become deprecated. */ - void recycle_to_reexecute() { - __TBB_ASSERT( prefix().state==executing, "execute not running, or already recycled" ); - __TBB_ASSERT( prefix().ref_count==0, "no child tasks allowed when recycled for reexecution" ); - prefix().state = reexecute; - } - -#if __TBB_RECYCLE_TO_ENQUEUE - //! Schedule this to enqueue after descendant tasks complete. - /** Save enqueue/spawn difference, it has the semantics of recycle_as_safe_continuation. */ - void recycle_to_enqueue() { - __TBB_ASSERT( prefix().state==executing, "execute not running, or already recycled" ); - prefix().state = to_enqueue; - } -#endif /* __TBB_RECYCLE_TO_ENQUEUE */ - - //------------------------------------------------------------------------ - // Spawning and blocking - //------------------------------------------------------------------------ - - //! Set reference count - void set_ref_count( int count ) { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - internal_set_ref_count(count); -#else - prefix().ref_count = count; -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - - //! Atomically increment reference count. - /** Has acquire semantics */ - void increment_ref_count() { - __TBB_FetchAndIncrementWacquire( &prefix().ref_count ); - } - - //! Atomically adds to reference count and returns its new value. - /** Has release-acquire semantics */ - int add_ref_count( int count ) { - internal::call_itt_notify( internal::releasing, &prefix().ref_count ); - internal::reference_count k = count+__TBB_FetchAndAddW( &prefix().ref_count, count ); - __TBB_ASSERT( k>=0, "task's reference count underflowed" ); - if( k==0 ) - internal::call_itt_notify( internal::acquired, &prefix().ref_count ); - return int(k); - } - - //! Atomically decrement reference count and returns its new value. - /** Has release semantics. */ - int decrement_ref_count() { -#if TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT - return int(internal_decrement_ref_count()); -#else - return int(__TBB_FetchAndDecrementWrelease( &prefix().ref_count ))-1; -#endif /* TBB_USE_THREADING_TOOLS||TBB_USE_ASSERT */ - } - - //! Define recommended static forms via import from base class. - using task_base::spawn; - - //! Similar to spawn followed by wait_for_all, but more efficient. - void spawn_and_wait_for_all( task& child ) { - prefix().owner->wait_for_all( *this, &child ); - } - - //! Similar to spawn followed by wait_for_all, but more efficient. - void __TBB_EXPORTED_METHOD spawn_and_wait_for_all( task_list& list ); - - //! Spawn task allocated by allocate_root, wait for it to complete, and deallocate it. - static void spawn_root_and_wait( task& root ) { - root.prefix().owner->spawn_root_and_wait( root, root.prefix().next ); - } - - //! Spawn root tasks on list and wait for all of them to finish. - /** If there are more tasks than worker threads, the tasks are spawned in - order of front to back. */ - static void spawn_root_and_wait( task_list& root_list ); - - //! Wait for reference count to become one, and set reference count to zero. - /** Works on tasks while waiting. */ - void wait_for_all() { - prefix().owner->wait_for_all( *this, NULL ); - } - - //! Enqueue task for starvation-resistant execution. -#if __TBB_TASK_PRIORITY - /** The task will be enqueued on the normal priority level disregarding the - priority of its task group. - - The rationale of such semantics is that priority of an enqueued task is - statically fixed at the moment of its enqueuing, while task group priority - is dynamic. Thus automatic priority inheritance would be generally a subject - to the race, which may result in unexpected behavior. - - Use enqueue() overload with explicit priority value and task::group_priority() - method to implement such priority inheritance when it is really necessary. **/ -#endif /* __TBB_TASK_PRIORITY */ - static void enqueue( task& t ) { - t.prefix().owner->enqueue( t, NULL ); - } - -#if __TBB_TASK_PRIORITY - //! Enqueue task for starvation-resistant execution on the specified priority level. - __TBB_DEPRECATED static void enqueue( task& t, priority_t p ) { -#if __TBB_PREVIEW_CRITICAL_TASKS - __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high - || p == internal::priority_critical, "Invalid priority level value"); -#else - __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high, "Invalid priority level value"); -#endif - t.prefix().owner->enqueue( t, (void*)p ); - } -#endif /* __TBB_TASK_PRIORITY */ - - //! Enqueue task in task_arena - //! The implementation is in task_arena.h -#if __TBB_TASK_PRIORITY - __TBB_DEPRECATED inline static void enqueue( task& t, task_arena& arena, priority_t p = priority_t(0) ); -#else - inline static void enqueue( task& t, task_arena& arena); -#endif - - //! The innermost task being executed or destroyed by the current thread at the moment. - static task& __TBB_EXPORTED_FUNC self(); - - //! task on whose behalf this task is working, or NULL if this is a root. - task* parent() const {return prefix().parent;} - - //! sets parent task pointer to specified value - void set_parent(task* p) { -#if __TBB_TASK_GROUP_CONTEXT - __TBB_ASSERT(!p || prefix().context == p->prefix().context, "The tasks must be in the same context"); -#endif - prefix().parent = p; - } - -#if __TBB_TASK_GROUP_CONTEXT - //! This method is deprecated and will be removed in the future. - /** Use method group() instead. **/ - task_group_context* context() {return prefix().context;} - - //! Pointer to the task group descriptor. - task_group_context* group () { return prefix().context; } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - - //! True if task was stolen from the task pool of another thread. - bool is_stolen_task() const { - return (prefix().extra_state & 0x80)!=0; - } - - //! True if the task was enqueued - bool is_enqueued_task() const { - // es_task_enqueued = 0x10 - return (prefix().extra_state & 0x10)!=0; - } - -#if __TBB_PREVIEW_RESUMABLE_TASKS - //! Type that defines suspension point - typedef void* suspend_point; - - //! Suspend current task execution - template - static void suspend(F f); - - //! Resume specific suspend point - static void resume(suspend_point tag); -#endif - - //------------------------------------------------------------------------ - // Debugging - //------------------------------------------------------------------------ - - //! Current execution state - state_type state() const {return state_type(prefix().state);} - - //! The internal reference count. - int ref_count() const { -#if TBB_USE_ASSERT -#if __TBB_PREVIEW_RESUMABLE_TASKS - internal::reference_count ref_count_ = prefix().ref_count & ~internal::abandon_flag; -#else - internal::reference_count ref_count_ = prefix().ref_count; -#endif - __TBB_ASSERT( ref_count_==int(ref_count_), "integer overflow error"); -#endif -#if __TBB_PREVIEW_RESUMABLE_TASKS - return int(prefix().ref_count & ~internal::abandon_flag); -#else - return int(prefix().ref_count); -#endif - } - - //! Obsolete, and only retained for the sake of backward compatibility. Always returns true. - bool __TBB_EXPORTED_METHOD is_owned_by_current_thread() const; - - //------------------------------------------------------------------------ - // Affinity - //------------------------------------------------------------------------ - - //! An id as used for specifying affinity. - /** Guaranteed to be integral type. Value of 0 means no affinity. */ - typedef internal::affinity_id affinity_id; - - //! Set affinity for this task. - void set_affinity( affinity_id id ) {prefix().affinity = id;} - - //! Current affinity of this task - affinity_id affinity() const {return prefix().affinity;} - - //! Invoked by scheduler to notify task that it ran on unexpected thread. - /** Invoked before method execute() runs, if task is stolen, or task has - affinity but will be executed on another thread. - - The default action does nothing. */ - virtual void __TBB_EXPORTED_METHOD note_affinity( affinity_id id ); - -#if __TBB_TASK_GROUP_CONTEXT - //! Moves this task from its current group into another one. - /** Argument ctx specifies the new group. - - The primary purpose of this method is to associate unique task group context - with a task allocated for subsequent enqueuing. In contrast to spawned tasks - enqueued ones normally outlive the scope where they were created. This makes - traditional usage model where task group context are allocated locally on - the stack inapplicable. Dynamic allocation of context objects is performance - inefficient. Method change_group() allows to make task group context object - a member of the task class, and then associate it with its containing task - object in the latter's constructor. **/ - void __TBB_EXPORTED_METHOD change_group ( task_group_context& ctx ); - - //! Initiates cancellation of all tasks in this cancellation group and its subordinate groups. - /** \return false if cancellation has already been requested, true otherwise. **/ - bool cancel_group_execution () { return prefix().context->cancel_group_execution(); } - - //! Returns true if the context has received cancellation request. - bool is_cancelled () const { return prefix().context->is_group_execution_cancelled(); } -#else - bool is_cancelled () const { return false; } -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -#if __TBB_TASK_PRIORITY - //! Changes priority of the task group this task belongs to. - __TBB_DEPRECATED void set_group_priority ( priority_t p ) { prefix().context->set_priority(p); } - - //! Retrieves current priority of the task group this task belongs to. - __TBB_DEPRECATED priority_t group_priority () const { return prefix().context->priority(); } - -#endif /* __TBB_TASK_PRIORITY */ - -private: - friend class interface5::internal::task_base; - friend class task_list; - friend class internal::scheduler; - friend class internal::allocate_root_proxy; -#if __TBB_TASK_GROUP_CONTEXT - friend class internal::allocate_root_with_context_proxy; -#endif /* __TBB_TASK_GROUP_CONTEXT */ - friend class internal::allocate_continuation_proxy; - friend class internal::allocate_child_proxy; - friend class internal::allocate_additional_child_of_proxy; - - //! Get reference to corresponding task_prefix. - /** Version tag prevents loader on Linux from using the wrong symbol in debug builds. **/ - internal::task_prefix& prefix( internal::version_tag* = NULL ) const { - return reinterpret_cast(const_cast(this))[-1]; - } -#if __TBB_PREVIEW_CRITICAL_TASKS - friend void internal::make_critical( task& ); - friend bool internal::is_critical( task& ); -#endif -}; // class task - -#if __TBB_PREVIEW_CRITICAL_TASKS -namespace internal { -inline void make_critical( task& t ) { t.prefix().extra_state |= 0x8; } -inline bool is_critical( task& t ) { return bool((t.prefix().extra_state & 0x8) != 0); } -} // namespace internal -#endif /* __TBB_PREVIEW_CRITICAL_TASKS */ - -#if __TBB_PREVIEW_RESUMABLE_TASKS -namespace internal { - template - static void suspend_callback(void* user_callback, task::suspend_point tag) { - // Copy user function to a new stack to avoid a race when the previous scheduler is resumed. - F user_callback_copy = *static_cast(user_callback); - user_callback_copy(tag); - } - void __TBB_EXPORTED_FUNC internal_suspend(void* suspend_callback, void* user_callback); - void __TBB_EXPORTED_FUNC internal_resume(task::suspend_point); - task::suspend_point __TBB_EXPORTED_FUNC internal_current_suspend_point(); -} - -template -inline void task::suspend(F f) { - internal::internal_suspend((void*)internal::suspend_callback, &f); -} -inline void task::resume(suspend_point tag) { - internal::internal_resume(tag); -} -#endif - -//! task that does nothing. Useful for synchronization. -/** @ingroup task_scheduling */ -class empty_task: public task { - task* execute() __TBB_override { - return NULL; - } -}; - -//! @cond INTERNAL -namespace internal { - template - class function_task : public task { -#if __TBB_ALLOW_MUTABLE_FUNCTORS - // TODO: deprecated behavior, remove - F my_func; -#else - const F my_func; -#endif - task* execute() __TBB_override { - my_func(); - return NULL; - } - public: - function_task( const F& f ) : my_func(f) {} -#if __TBB_CPP11_RVALUE_REF_PRESENT - function_task( F&& f ) : my_func( std::move(f) ) {} -#endif - }; -} // namespace internal -//! @endcond - -//! A list of children. -/** Used for method task::spawn_children - @ingroup task_scheduling */ -class task_list: internal::no_copy { -private: - task* first; - task** next_ptr; - friend class task; - friend class interface5::internal::task_base; -public: - //! Construct empty list - task_list() : first(NULL), next_ptr(&first) {} - - //! Destroys the list, but does not destroy the task objects. - ~task_list() {} - - //! True if list is empty; false otherwise. - bool empty() const {return !first;} - - //! Push task onto back of list. - void push_back( task& task ) { - task.prefix().next = NULL; - *next_ptr = &task; - next_ptr = &task.prefix().next; - } -#if __TBB_TODO - // TODO: add this method and implement&document the local execution ordering. See more in generic_scheduler::local_spawn - //! Push task onto front of list (FIFO local execution, like individual spawning in the same order). - void push_front( task& task ) { - if( empty() ) { - push_back(task); - } else { - task.prefix().next = first; - first = &task; - } - } -#endif - //! Pop the front task from the list. - task& pop_front() { - __TBB_ASSERT( !empty(), "attempt to pop item from empty task_list" ); - task* result = first; - first = result->prefix().next; - if( !first ) next_ptr = &first; - return *result; - } - - //! Clear the list - void clear() { - first=NULL; - next_ptr=&first; - } -}; - -inline void interface5::internal::task_base::spawn( task& t ) { - t.prefix().owner->spawn( t, t.prefix().next ); -} - -inline void interface5::internal::task_base::spawn( task_list& list ) { - if( task* t = list.first ) { - t->prefix().owner->spawn( *t, *list.next_ptr ); - list.clear(); - } -} - -inline void task::spawn_root_and_wait( task_list& root_list ) { - if( task* t = root_list.first ) { - t->prefix().owner->spawn_root_and_wait( *t, *root_list.next_ptr ); - root_list.clear(); - } -} - -} // namespace tbb - -inline void *operator new( size_t bytes, const tbb::internal::allocate_root_proxy& ) { - return &tbb::internal::allocate_root_proxy::allocate(bytes); -} - -inline void operator delete( void* task, const tbb::internal::allocate_root_proxy& ) { - tbb::internal::allocate_root_proxy::free( *static_cast(task) ); -} - -#if __TBB_TASK_GROUP_CONTEXT -inline void *operator new( size_t bytes, const tbb::internal::allocate_root_with_context_proxy& p ) { - return &p.allocate(bytes); -} - -inline void operator delete( void* task, const tbb::internal::allocate_root_with_context_proxy& p ) { - p.free( *static_cast(task) ); -} -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -inline void *operator new( size_t bytes, const tbb::internal::allocate_continuation_proxy& p ) { - return &p.allocate(bytes); -} - -inline void operator delete( void* task, const tbb::internal::allocate_continuation_proxy& p ) { - p.free( *static_cast(task) ); -} - -inline void *operator new( size_t bytes, const tbb::internal::allocate_child_proxy& p ) { - return &p.allocate(bytes); -} - -inline void operator delete( void* task, const tbb::internal::allocate_child_proxy& p ) { - p.free( *static_cast(task) ); -} - -inline void *operator new( size_t bytes, const tbb::internal::allocate_additional_child_of_proxy& p ) { - return &p.allocate(bytes); -} - -inline void operator delete( void* task, const tbb::internal::allocate_additional_child_of_proxy& p ) { - p.free( *static_cast(task) ); -} - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_task_H_include_area - -#endif /* __TBB_task_H */ +#include "../oneapi/tbb/task.h" diff --git a/src/3rd party/tbb/task_arena.h b/src/3rd party/tbb/task_arena.h index 83ba9118e4..f6e34b3e6d 100644 --- a/src/3rd party/tbb/task_arena.h +++ b/src/3rd party/tbb/task_arena.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,493 +14,4 @@ limitations under the License. */ -#ifndef __TBB_task_arena_H -#define __TBB_task_arena_H - -#define __TBB_task_arena_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "task.h" -#include "tbb_exception.h" -#include "internal/_template_helpers.h" -#if __TBB_NUMA_SUPPORT -#include "info.h" -#endif /*__TBB_NUMA_SUPPORT*/ -#if TBB_USE_THREADING_TOOLS -#include "atomic.h" // for as_atomic -#endif -#include "aligned_space.h" - -namespace tbb { - -namespace this_task_arena { - int max_concurrency(); -} // namespace this_task_arena - -//! @cond INTERNAL -namespace internal { - //! Internal to library. Should not be used by clients. - /** @ingroup task_scheduling */ - class arena; - class task_scheduler_observer_v3; -} // namespace internal -//! @endcond - -namespace interface7 { -class task_arena; - -//! @cond INTERNAL -namespace internal { -using namespace tbb::internal; //e.g. function_task from task.h - -class delegate_base : no_assign { -public: - virtual void operator()() const = 0; - virtual ~delegate_base() {} -}; - -// If decltype is available, the helper detects the return type of functor of specified type, -// otherwise it defines the void type. -template -struct return_type_or_void { -#if __TBB_CPP11_DECLTYPE_PRESENT && !__TBB_CPP11_DECLTYPE_OF_FUNCTION_RETURN_TYPE_BROKEN - typedef decltype(declval()()) type; -#else - typedef void type; -#endif -}; - -template -class delegated_function : public delegate_base { - F &my_func; - tbb::aligned_space my_return_storage; - // The function should be called only once. - void operator()() const __TBB_override { - new (my_return_storage.begin()) R(my_func()); - } -public: - delegated_function(F& f) : my_func(f) {} - // The function can be called only after operator() and only once. - R consume_result() const { - return tbb::internal::move(*(my_return_storage.begin())); - } - ~delegated_function() { - my_return_storage.begin()->~R(); - } -}; - -template -class delegated_function : public delegate_base { - F &my_func; - void operator()() const __TBB_override { - my_func(); - } -public: - delegated_function(F& f) : my_func(f) {} - void consume_result() const {} - - friend class task_arena_base; -}; - -class task_arena_base { -#if __TBB_NUMA_SUPPORT -public: - // TODO: consider version approach to resolve backward compatibility potential issues. - struct constraints { - constraints(numa_node_id id = automatic, int maximal_concurrency = automatic) - : numa_id(id) - , max_concurrency(maximal_concurrency) - {} - numa_node_id numa_id; - int max_concurrency; - }; -#endif /*__TBB_NUMA_SUPPORT*/ -protected: - //! NULL if not currently initialized. - internal::arena* my_arena; - -#if __TBB_TASK_GROUP_CONTEXT - //! default context of the arena - task_group_context *my_context; -#endif - - //! Concurrency level for deferred initialization - int my_max_concurrency; - - //! Reserved master slots - unsigned my_master_slots; - - //! Special settings - intptr_t my_version_and_traits; - - bool my_initialized; - -#if __TBB_NUMA_SUPPORT - //! The NUMA node index to which the arena will be attached - numa_node_id my_numa_id; - - // Do not access my_numa_id without the following runtime check. - // Despite my_numa_id is accesible, it does not exist in task_arena_base on user side - // if TBB_PREVIEW_NUMA_SUPPORT macro is not defined by the user. To be sure that - // my_numa_id exists in task_arena_base layout we check the traits. - // TODO: Consider increasing interface version for task_arena_base instead of this runtime check. - numa_node_id numa_id() { - return (my_version_and_traits & numa_support_flag) == numa_support_flag ? my_numa_id : automatic; - } -#endif - - enum { - default_flags = 0 -#if __TBB_TASK_GROUP_CONTEXT - | (task_group_context::default_traits & task_group_context::exact_exception) // 0 or 1 << 16 - , exact_exception_flag = task_group_context::exact_exception // used to specify flag for context directly -#endif -#if __TBB_NUMA_SUPPORT - , numa_support_flag = 1 -#endif - }; - - task_arena_base(int max_concurrency, unsigned reserved_for_masters) - : my_arena(0) -#if __TBB_TASK_GROUP_CONTEXT - , my_context(0) -#endif - , my_max_concurrency(max_concurrency) - , my_master_slots(reserved_for_masters) -#if __TBB_NUMA_SUPPORT - , my_version_and_traits(default_flags | numa_support_flag) -#else - , my_version_and_traits(default_flags) -#endif - , my_initialized(false) -#if __TBB_NUMA_SUPPORT - , my_numa_id(automatic) -#endif - {} - -#if __TBB_NUMA_SUPPORT - task_arena_base(const constraints& constraints_, unsigned reserved_for_masters) - : my_arena(0) -#if __TBB_TASK_GROUP_CONTEXT - , my_context(0) -#endif - , my_max_concurrency(constraints_.max_concurrency) - , my_master_slots(reserved_for_masters) - , my_version_and_traits(default_flags | numa_support_flag) - , my_initialized(false) - , my_numa_id(constraints_.numa_id ) - {} -#endif /*__TBB_NUMA_SUPPORT*/ - - void __TBB_EXPORTED_METHOD internal_initialize(); - void __TBB_EXPORTED_METHOD internal_terminate(); - void __TBB_EXPORTED_METHOD internal_attach(); - void __TBB_EXPORTED_METHOD internal_enqueue( task&, intptr_t ) const; - void __TBB_EXPORTED_METHOD internal_execute( delegate_base& ) const; - void __TBB_EXPORTED_METHOD internal_wait() const; - static int __TBB_EXPORTED_FUNC internal_current_slot(); - static int __TBB_EXPORTED_FUNC internal_max_concurrency( const task_arena * ); -public: - //! Typedef for number of threads that is automatic. - static const int automatic = -1; - static const int not_initialized = -2; - -}; - -#if __TBB_TASK_ISOLATION -void __TBB_EXPORTED_FUNC isolate_within_arena( delegate_base& d, intptr_t isolation = 0 ); - -template -R isolate_impl(F& f) { - delegated_function d(f); - isolate_within_arena(d); - return d.consume_result(); -} -#endif /* __TBB_TASK_ISOLATION */ -} // namespace internal -//! @endcond - -/** 1-to-1 proxy representation class of scheduler's arena - * Constructors set up settings only, real construction is deferred till the first method invocation - * Destructor only removes one of the references to the inner arena representation. - * Final destruction happens when all the references (and the work) are gone. - */ -class task_arena : public internal::task_arena_base { - friend class tbb::internal::task_scheduler_observer_v3; - friend void task::enqueue(task&, task_arena& -#if __TBB_TASK_PRIORITY - , priority_t -#endif - ); - friend int tbb::this_task_arena::max_concurrency(); - void mark_initialized() { - __TBB_ASSERT( my_arena, "task_arena initialization is incomplete" ); -#if __TBB_TASK_GROUP_CONTEXT - __TBB_ASSERT( my_context, "task_arena initialization is incomplete" ); -#endif -#if TBB_USE_THREADING_TOOLS - // Actual synchronization happens in internal_initialize & internal_attach. - // The race on setting my_initialized is benign, but should be hidden from Intel(R) Inspector - internal::as_atomic(my_initialized).fetch_and_store(true); -#else - my_initialized = true; -#endif - } - - template - void enqueue_impl( __TBB_FORWARDING_REF(F) f -#if __TBB_TASK_PRIORITY - , priority_t p = priority_t(0) -#endif - ) { -#if !__TBB_TASK_PRIORITY - intptr_t p = 0; -#endif - initialize(); -#if __TBB_TASK_GROUP_CONTEXT - internal_enqueue(*new(task::allocate_root(*my_context)) internal::function_task< typename internal::strip::type >(internal::forward(f)), p); -#else - internal_enqueue(*new(task::allocate_root()) internal::function_task< typename internal::strip::type >(internal::forward(f)), p); -#endif /* __TBB_TASK_GROUP_CONTEXT */ - } - - template - R execute_impl(F& f) { - initialize(); - internal::delegated_function d(f); - internal_execute(d); - return d.consume_result(); - } - -public: - //! Creates task_arena with certain concurrency limits - /** Sets up settings only, real construction is deferred till the first method invocation - * @arg max_concurrency specifies total number of slots in arena where threads work - * @arg reserved_for_masters specifies number of slots to be used by master threads only. - * Value of 1 is default and reflects behavior of implicit arenas. - **/ - task_arena(int max_concurrency_ = automatic, unsigned reserved_for_masters = 1) - : task_arena_base(max_concurrency_, reserved_for_masters) - {} - -#if __TBB_NUMA_SUPPORT - //! Creates task arena pinned to certain NUMA node - task_arena(const constraints& constraints_, unsigned reserved_for_masters = 1) - : task_arena_base(constraints_, reserved_for_masters) - {} -#endif /*__TBB_NUMA_SUPPORT*/ - - //! Copies settings from another task_arena - task_arena(const task_arena &s) // copy settings but not the reference or instance - : task_arena_base(s.my_max_concurrency, s.my_master_slots) - {} - - //! Tag class used to indicate the "attaching" constructor - struct attach {}; - - //! Creates an instance of task_arena attached to the current arena of the thread - explicit task_arena( attach ) - : task_arena_base(automatic, 1) // use default settings if attach fails - { - internal_attach(); - if( my_arena ) my_initialized = true; - } - - //! Forces allocation of the resources for the task_arena as specified in constructor arguments - inline void initialize() { - if( !my_initialized ) { - internal_initialize(); - mark_initialized(); - } - } - - //! Overrides concurrency level and forces initialization of internal representation - inline void initialize(int max_concurrency_, unsigned reserved_for_masters = 1) { - // TODO: decide if this call must be thread-safe - __TBB_ASSERT(!my_arena, "Impossible to modify settings of an already initialized task_arena"); - if( !my_initialized ) { - my_max_concurrency = max_concurrency_; - my_master_slots = reserved_for_masters; - initialize(); - } - } - -#if __TBB_NUMA_SUPPORT - inline void initialize(constraints constraints_, unsigned reserved_for_masters = 1) { - // TODO: decide if this call must be thread-safe - __TBB_ASSERT(!my_arena, "Impossible to modify settings of an already initialized task_arena"); - if( !my_initialized ) { - my_numa_id = constraints_.numa_id; - my_max_concurrency = constraints_.max_concurrency; - my_master_slots = reserved_for_masters; - initialize(); - } - } -#endif /*__TBB_NUMA_SUPPORT*/ - - //! Attaches this instance to the current arena of the thread - inline void initialize(attach) { - // TODO: decide if this call must be thread-safe - __TBB_ASSERT(!my_arena, "Impossible to modify settings of an already initialized task_arena"); - if( !my_initialized ) { - internal_attach(); - if ( !my_arena ) internal_initialize(); - mark_initialized(); - } - } - - //! Removes the reference to the internal arena representation. - //! Not thread safe wrt concurrent invocations of other methods. - inline void terminate() { - if( my_initialized ) { - internal_terminate(); - my_initialized = false; - } - } - - //! Removes the reference to the internal arena representation, and destroys the external object. - //! Not thread safe wrt concurrent invocations of other methods. - ~task_arena() { - terminate(); - } - - //! Returns true if the arena is active (initialized); false otherwise. - //! The name was chosen to match a task_scheduler_init method with the same semantics. - bool is_active() const { return my_initialized; } - - //! Enqueues a task into the arena to process a functor, and immediately returns. - //! Does not require the calling thread to join the arena - -#if __TBB_CPP11_RVALUE_REF_PRESENT - template - void enqueue( F&& f ) { - enqueue_impl(std::forward(f)); - } -#else - template - void enqueue( const F& f ) { - enqueue_impl(f); - } -#endif - -#if __TBB_TASK_PRIORITY - //! Enqueues a task with priority p into the arena to process a functor f, and immediately returns. - //! Does not require the calling thread to join the arena - template -#if __TBB_CPP11_RVALUE_REF_PRESENT - __TBB_DEPRECATED void enqueue( F&& f, priority_t p ) { -#if __TBB_PREVIEW_CRITICAL_TASKS - __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high - || p == internal::priority_critical, "Invalid priority level value"); -#else - __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high, "Invalid priority level value"); -#endif - enqueue_impl(std::forward(f), p); - } -#else - __TBB_DEPRECATED void enqueue( const F& f, priority_t p ) { -#if __TBB_PREVIEW_CRITICAL_TASKS - __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high - || p == internal::priority_critical, "Invalid priority level value"); -#else - __TBB_ASSERT(p == priority_low || p == priority_normal || p == priority_high, "Invalid priority level value"); -#endif - enqueue_impl(f,p); - } -#endif -#endif// __TBB_TASK_PRIORITY - - //! Joins the arena and executes a mutable functor, then returns - //! If not possible to join, wraps the functor into a task, enqueues it and waits for task completion - //! Can decrement the arena demand for workers, causing a worker to leave and free a slot to the calling thread - //! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void). - template - typename internal::return_type_or_void::type execute(F& f) { - return execute_impl::type>(f); - } - - //! Joins the arena and executes a constant functor, then returns - //! If not possible to join, wraps the functor into a task, enqueues it and waits for task completion - //! Can decrement the arena demand for workers, causing a worker to leave and free a slot to the calling thread - //! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void). - template - typename internal::return_type_or_void::type execute(const F& f) { - return execute_impl::type>(f); - } - -#if __TBB_EXTRA_DEBUG - //! Wait for all work in the arena to be completed - //! Even submitted by other application threads - //! Joins arena if/when possible (in the same way as execute()) - void debug_wait_until_empty() { - initialize(); - internal_wait(); - } -#endif //__TBB_EXTRA_DEBUG - - //! Returns the index, aka slot number, of the calling thread in its current arena - //! This method is deprecated and replaced with this_task_arena::current_thread_index() - inline static int current_thread_index() { - return internal_current_slot(); - } - - //! Returns the maximal number of threads that can work inside the arena - inline int max_concurrency() const { - // Handle special cases inside the library - return (my_max_concurrency>1) ? my_max_concurrency : internal_max_concurrency(this); - } -}; - -namespace this_task_arena { -#if __TBB_TASK_ISOLATION - //! Executes a mutable functor in isolation within the current task arena. - //! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void). - template - typename internal::return_type_or_void::type isolate(F& f) { - return internal::isolate_impl::type>(f); - } - - //! Executes a constant functor in isolation within the current task arena. - //! Since C++11, the method returns the value returned by functor (prior to C++11 it returns void). - template - typename internal::return_type_or_void::type isolate(const F& f) { - return internal::isolate_impl::type>(f); - } -#endif /* __TBB_TASK_ISOLATION */ -} // namespace this_task_arena -} // namespace interfaceX - -using interface7::task_arena; - -namespace this_task_arena { - using namespace interface7::this_task_arena; - - //! Returns the index, aka slot number, of the calling thread in its current arena - inline int current_thread_index() { - int idx = tbb::task_arena::current_thread_index(); - return idx == -1 ? tbb::task_arena::not_initialized : idx; - } - - //! Returns the maximal number of threads that can work inside the arena - inline int max_concurrency() { - return tbb::task_arena::internal_max_concurrency(NULL); - } -} // namespace this_task_arena - -//! Enqueue task in task_arena -#if __TBB_TASK_PRIORITY -void task::enqueue( task& t, task_arena& arena, priority_t p ) { -#else -void task::enqueue( task& t, task_arena& arena ) { - intptr_t p = 0; -#endif - arena.initialize(); - //! Note: the context of the task may differ from the context instantiated by task_arena - arena.internal_enqueue(t, p); -} -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_task_arena_H_include_area - -#endif /* __TBB_task_arena_H */ +#include "../oneapi/tbb/task_arena.h" diff --git a/src/3rd party/tbb/task_group.h b/src/3rd party/tbb/task_group.h index cb16500e86..2f02503971 100644 --- a/src/3rd party/tbb/task_group.h +++ b/src/3rd party/tbb/task_group.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,353 +14,4 @@ limitations under the License. */ -#ifndef __TBB_task_group_H -#define __TBB_task_group_H - -#define __TBB_task_group_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "task.h" -#include "tbb_exception.h" -#include "internal/_template_helpers.h" -#if TBB_PREVIEW_ISOLATED_TASK_GROUP && __TBB_TASK_ISOLATION -#include "task_arena.h" -#endif - -#if __TBB_TASK_GROUP_CONTEXT - -namespace tbb { - -namespace internal { - template class task_handle_task; -} - -class task_group; -class structured_task_group; -#if TBB_PREVIEW_ISOLATED_TASK_GROUP && __TBB_TASK_ISOLATION -class isolated_task_group; -#endif - -template -class task_handle : internal::no_assign { - template friend class internal::task_handle_task; - friend class task_group; - friend class structured_task_group; -#if TBB_PREVIEW_ISOLATED_TASK_GROUP && __TBB_TASK_ISOLATION - friend class isolated_task_group; -#endif - - static const intptr_t scheduled = 0x1; - - F my_func; - intptr_t my_state; - - void mark_scheduled () { - // The check here is intentionally lax to avoid the impact of interlocked operation - if ( my_state & scheduled ) - internal::throw_exception( internal::eid_invalid_multiple_scheduling ); - my_state |= scheduled; - } -public: - task_handle( const F& f ) : my_func(f), my_state(0) {} -#if __TBB_CPP11_RVALUE_REF_PRESENT - task_handle( F&& f ) : my_func( std::move(f)), my_state(0) {} -#endif - - void operator() () const { my_func(); } -}; - -enum task_group_status { - not_complete, - complete, - canceled -}; - -namespace internal { - -template -class task_handle_task : public task { - task_handle& my_handle; - task* execute() __TBB_override { - my_handle(); - return NULL; - } -public: - task_handle_task( task_handle& h ) : my_handle(h) { h.mark_scheduled(); } -}; - -class task_group_base : internal::no_copy { - class ref_count_guard : internal::no_copy { - task& my_task; - public: - ref_count_guard(task& t) : my_task(t) { - my_task.increment_ref_count(); - } - ~ref_count_guard() { - my_task.decrement_ref_count(); - } - }; -protected: - empty_task* my_root; - task_group_context my_context; - - template - task_group_status internal_run_and_wait( F& f ) { - __TBB_TRY { - if ( !my_context.is_group_execution_cancelled() ) { - // We need to increase the reference count of the root task to notify waiters that - // this task group has some work in progress. - ref_count_guard guard(*my_root); - f(); - } - } __TBB_CATCH( ... ) { - my_context.register_pending_exception(); - } - return wait(); - } - - template - task* prepare_task( __TBB_FORWARDING_REF(F) f ) { - return new( task::allocate_additional_child_of(*my_root) ) Task( internal::forward(f) ); - } - -public: - task_group_base( uintptr_t traits = 0 ) - : my_context(task_group_context::bound, task_group_context::default_traits | traits) - { - my_root = new( task::allocate_root(my_context) ) empty_task; - my_root->set_ref_count(1); - } - - ~task_group_base() __TBB_NOEXCEPT(false) { - if( my_root->ref_count() > 1 ) { -#if __TBB_CPP17_UNCAUGHT_EXCEPTIONS_PRESENT - bool stack_unwinding_in_progress = std::uncaught_exceptions() > 0; -#else - bool stack_unwinding_in_progress = std::uncaught_exception(); -#endif - // Always attempt to do proper cleanup to avoid inevitable memory corruption - // in case of missing wait (for the sake of better testability & debuggability) - if ( !is_canceling() ) - cancel(); - __TBB_TRY { - my_root->wait_for_all(); - } __TBB_CATCH (...) { - task::destroy(*my_root); - __TBB_RETHROW(); - } - task::destroy(*my_root); - if ( !stack_unwinding_in_progress ) - internal::throw_exception( internal::eid_missing_wait ); - } - else { - task::destroy(*my_root); - } - } - - template - void run( task_handle& h ) { - task::spawn( *prepare_task< internal::task_handle_task >(h) ); - } - - task_group_status wait() { - __TBB_TRY { - my_root->wait_for_all(); - } __TBB_CATCH( ... ) { - my_context.reset(); - __TBB_RETHROW(); - } - if ( my_context.is_group_execution_cancelled() ) { - // TODO: the reset method is not thread-safe. Ensure the correct behavior. - my_context.reset(); - return canceled; - } - return complete; - } - - bool is_canceling() { - return my_context.is_group_execution_cancelled(); - } - - void cancel() { - my_context.cancel_group_execution(); - } -}; // class task_group_base - -} // namespace internal - -class task_group : public internal::task_group_base { -public: - task_group () : task_group_base( task_group_context::concurrent_wait ) {} - -#if __SUNPRO_CC - template - void run( task_handle& h ) { - internal_run< internal::task_handle_task >( h ); - } -#else - using task_group_base::run; -#endif - -#if __TBB_CPP11_RVALUE_REF_PRESENT - template - void run( F&& f ) { - task::spawn( *prepare_task< internal::function_task< typename internal::strip::type > >(std::forward(f)) ); - } -#else - template - void run(const F& f) { - task::spawn( *prepare_task< internal::function_task >(f) ); - } -#endif - - template - task_group_status run_and_wait( const F& f ) { - return internal_run_and_wait( f ); - } - - // TODO: add task_handle rvalues support - template - task_group_status run_and_wait( task_handle& h ) { - h.mark_scheduled(); - return internal_run_and_wait< task_handle >( h ); - } -}; // class task_group - -class __TBB_DEPRECATED structured_task_group : public internal::task_group_base { -public: - // TODO: add task_handle rvalues support - template - task_group_status run_and_wait ( task_handle& h ) { - h.mark_scheduled(); - return internal_run_and_wait< task_handle >( h ); - } - - task_group_status wait() { - task_group_status res = task_group_base::wait(); - my_root->set_ref_count(1); - return res; - } -}; // class structured_task_group - -#if TBB_PREVIEW_ISOLATED_TASK_GROUP && __TBB_TASK_ISOLATION -namespace internal { - using interface7::internal::delegate_base; - using interface7::internal::isolate_within_arena; - - class spawn_delegate : public delegate_base { - task* task_to_spawn; - void operator()() const __TBB_override { - task::spawn(*task_to_spawn); - } - public: - spawn_delegate(task* a_task) : task_to_spawn(a_task) {} - }; - - class wait_delegate : public delegate_base { - void operator()() const __TBB_override { - status = tg.wait(); - } - protected: - task_group& tg; - task_group_status& status; - public: - wait_delegate(task_group& a_group, task_group_status& tgs) - : tg(a_group), status(tgs) {} - }; - - template - class run_wait_delegate : public wait_delegate { - F& func; - void operator()() const __TBB_override { - status = tg.run_and_wait( func ); - } - public: - run_wait_delegate(task_group& a_group, F& a_func, task_group_status& tgs) - : wait_delegate(a_group, tgs), func(a_func) {} - }; -} // namespace internal - -class isolated_task_group : public task_group { - intptr_t this_isolation() { - return reinterpret_cast(this); - } -public: - isolated_task_group () : task_group() {} - -#if __TBB_CPP11_RVALUE_REF_PRESENT - template - void run( F&& f ) { - internal::spawn_delegate sd( - prepare_task< internal::function_task< typename internal::strip::type > >(std::forward(f)) - ); - internal::isolate_within_arena( sd, this_isolation() ); - } -#else - template - void run(const F& f) { - internal::spawn_delegate sd( prepare_task< internal::function_task >(f) ); - internal::isolate_within_arena( sd, this_isolation() ); - } -#endif - - template - task_group_status run_and_wait( const F& f ) { - task_group_status result = not_complete; - internal::run_wait_delegate< const F > rwd( *this, f, result ); - internal::isolate_within_arena( rwd, this_isolation() ); - __TBB_ASSERT( result!=not_complete, "premature exit from wait?" ); - return result; - } - - // TODO: add task_handle rvalues support - template - void run( task_handle& h ) { - internal::spawn_delegate sd( prepare_task< internal::task_handle_task >(h) ); - internal::isolate_within_arena( sd, this_isolation() ); - } - - template - task_group_status run_and_wait ( task_handle& h ) { - task_group_status result = not_complete; - internal::run_wait_delegate< task_handle > rwd( *this, h, result ); - internal::isolate_within_arena( rwd, this_isolation() ); - __TBB_ASSERT( result!=not_complete, "premature exit from wait?" ); - return result; - } - - task_group_status wait() { - task_group_status result = not_complete; - internal::wait_delegate wd( *this, result ); - internal::isolate_within_arena( wd, this_isolation() ); - __TBB_ASSERT( result!=not_complete, "premature exit from wait?" ); - return result; - } -}; // class isolated_task_group -#endif // TBB_PREVIEW_ISOLATED_TASK_GROUP && __TBB_TASK_ISOLATION - -inline -bool is_current_task_group_canceling() { - return task::self().is_cancelled(); -} - -#if __TBB_CPP11_RVALUE_REF_PRESENT -template -task_handle< typename internal::strip::type > make_task( F&& f ) { - return task_handle< typename internal::strip::type >( std::forward(f) ); -} -#else -template -task_handle make_task( const F& f ) { - return task_handle( f ); -} -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - -} // namespace tbb - -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_task_group_H_include_area - -#endif /* __TBB_task_group_H */ +#include "../oneapi/tbb/task_group.h" diff --git a/src/3rd party/tbb/task_scheduler_init.h b/src/3rd party/tbb/task_scheduler_init.h deleted file mode 100644 index 3f9f7b4978..0000000000 --- a/src/3rd party/tbb/task_scheduler_init.h +++ /dev/null @@ -1,174 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include "internal/_deprecated_header_message_guard.h" - -#if !defined(__TBB_show_deprecation_message_task_scheduler_init_H) && defined(__TBB_show_deprecated_header_message) -#define __TBB_show_deprecation_message_task_scheduler_init_H -#pragma message("TBB Warning: tbb/task_scheduler_init.h is deprecated. For details, please see Deprecated Features appendix in the TBB reference manual.") -#endif - -#if defined(__TBB_show_deprecated_header_message) -#undef __TBB_show_deprecated_header_message -#endif - -#ifndef __TBB_task_scheduler_init_H -#define __TBB_task_scheduler_init_H - -#define __TBB_task_scheduler_init_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "tbb_stddef.h" -#include "limits.h" -#if __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE -#include // nothrow_t -#endif - -namespace tbb { - -typedef std::size_t stack_size_type; - -//! @cond INTERNAL -namespace internal { - //! Internal to library. Should not be used by clients. - /** @ingroup task_scheduling */ - class scheduler; -} // namespace internal -//! @endcond - -//! Class delimiting the scope of task scheduler activity. -/** A thread can construct a task_scheduler_init object and keep it alive - while it uses TBB's tasking subsystem (including parallel algorithms). - - This class allows to customize properties of the TBB task pool to some extent. - For example it can limit concurrency level of parallel work initiated by the - given thread. It also can be used to specify stack size of the TBB worker threads, - though this setting is not effective if the thread pool has already been created. - - If a parallel construct is used without task_scheduler_init object previously - created, the scheduler will be initialized automatically with default settings, - and will persist until this thread exits. Default concurrency level is defined - as described in task_scheduler_init::initialize(). - @ingroup task_scheduling */ -class __TBB_DEPRECATED_VERBOSE task_scheduler_init: internal::no_copy { - enum ExceptionPropagationMode { - propagation_mode_exact = 1u, - propagation_mode_captured = 2u, - propagation_mode_mask = propagation_mode_exact | propagation_mode_captured - }; - - /** NULL if not currently initialized. */ - internal::scheduler* my_scheduler; - - bool internal_terminate( bool blocking ); -#if __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE - bool __TBB_EXPORTED_METHOD internal_blocking_terminate( bool throwing ); -#endif -public: - - //! Typedef for number of threads that is automatic. - static const int automatic = -1; - - //! Argument to initialize() or constructor that causes initialization to be deferred. - static const int deferred = -2; - - //! Ensure that scheduler exists for this thread - /** A value of -1 lets TBB decide on the number of threads, which is usually - maximal hardware concurrency for this process, that is the number of logical - CPUs on the machine (possibly limited by the processor affinity mask of this - process (Windows) or of this thread (Linux, FreeBSD). It is preferable option - for production code because it helps to avoid nasty surprises when several - TBB based components run side-by-side or in a nested fashion inside the same - process. - - The number_of_threads is ignored if any other task_scheduler_inits - currently exist. A thread may construct multiple task_scheduler_inits. - Doing so does no harm because the underlying scheduler is reference counted. */ - void __TBB_EXPORTED_METHOD initialize( int number_of_threads=automatic ); - - //! The overloaded method with stack size parameter - /** Overloading is necessary to preserve ABI compatibility */ - void __TBB_EXPORTED_METHOD initialize( int number_of_threads, stack_size_type thread_stack_size ); - - //! Inverse of method initialize. - void __TBB_EXPORTED_METHOD terminate(); - -#if __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE -#if TBB_USE_EXCEPTIONS - //! terminate() that waits for worker threads termination. Throws exception on error. - void blocking_terminate() { - internal_blocking_terminate( /*throwing=*/true ); - } -#endif - //! terminate() that waits for worker threads termination. Returns false on error. - bool blocking_terminate(const std::nothrow_t&) __TBB_NOEXCEPT(true) { - return internal_blocking_terminate( /*throwing=*/false ); - } -#endif // __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE - - //! Shorthand for default constructor followed by call to initialize(number_of_threads). - task_scheduler_init( int number_of_threads=automatic, stack_size_type thread_stack_size=0 ) : my_scheduler(NULL) - { - // Two lowest order bits of the stack size argument may be taken to communicate - // default exception propagation mode of the client to be used when the - // client manually creates tasks in the master thread and does not use - // explicit task group context object. This is necessary because newer - // TBB binaries with exact propagation enabled by default may be used - // by older clients that expect tbb::captured_exception wrapper. - // All zeros mean old client - no preference. - __TBB_ASSERT( !(thread_stack_size & propagation_mode_mask), "Requested stack size is not aligned" ); -#if TBB_USE_EXCEPTIONS - thread_stack_size |= TBB_USE_CAPTURED_EXCEPTION ? propagation_mode_captured : propagation_mode_exact; -#endif /* TBB_USE_EXCEPTIONS */ - initialize( number_of_threads, thread_stack_size ); - } - - //! Destroy scheduler for this thread if thread has no other live task_scheduler_inits. - ~task_scheduler_init() { - if( my_scheduler ) - terminate(); - internal::poison_pointer( my_scheduler ); - } - //! Returns the number of threads TBB scheduler would create if initialized by default. - /** Result returned by this method does not depend on whether the scheduler - has already been initialized. - - Because tbb 2.0 does not support blocking tasks yet, you may use this method - to boost the number of threads in the tbb's internal pool, if your tasks are - doing I/O operations. The optimal number of additional threads depends on how - much time your tasks spend in the blocked state. - - Before TBB 3.0 U4 this method returned the number of logical CPU in the - system. Currently on Windows, Linux and FreeBSD it returns the number of - logical CPUs available to the current process in accordance with its affinity - mask. - - NOTE: The return value of this method never changes after its first invocation. - This means that changes in the process affinity mask that took place after - this method was first invoked will not affect the number of worker threads - in the TBB worker threads pool. */ - static int __TBB_EXPORTED_FUNC default_num_threads (); - - //! Returns true if scheduler is active (initialized); false otherwise - bool is_active() const { return my_scheduler != NULL; } -}; - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_task_scheduler_init_H_include_area - -#endif /* __TBB_task_scheduler_init_H */ diff --git a/src/3rd party/tbb/task_scheduler_observer.h b/src/3rd party/tbb/task_scheduler_observer.h index 76a452928f..9236f4cdf4 100644 --- a/src/3rd party/tbb/task_scheduler_observer.h +++ b/src/3rd party/tbb/task_scheduler_observer.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,153 +14,4 @@ limitations under the License. */ -#ifndef __TBB_task_scheduler_observer_H -#define __TBB_task_scheduler_observer_H - -#define __TBB_task_scheduler_observer_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "atomic.h" -#if __TBB_ARENA_OBSERVER -#include "task_arena.h" -#endif - -#if __TBB_SCHEDULER_OBSERVER - -namespace tbb { -namespace interface6 { -class task_scheduler_observer; -} -namespace internal { - -class observer_proxy; -class observer_list; - -class task_scheduler_observer_v3 { - friend class observer_proxy; - friend class observer_list; - friend class interface6::task_scheduler_observer; - - //! Pointer to the proxy holding this observer. - /** Observers are proxied by the scheduler to maintain persistent lists of them. **/ - observer_proxy* my_proxy; - - //! Counter preventing the observer from being destroyed while in use by the scheduler. - /** Valid only when observation is on. **/ - atomic my_busy_count; - -public: - //! Enable or disable observation - /** For local observers the method can be used only when the current thread - has the task scheduler initialized or is attached to an arena. - - Repeated calls with the same state are no-ops. **/ - void __TBB_EXPORTED_METHOD observe( bool state=true ); - - //! Returns true if observation is enabled, false otherwise. - bool is_observing() const {return my_proxy!=NULL;} - - //! Construct observer with observation disabled. - task_scheduler_observer_v3() : my_proxy(NULL) { my_busy_count.store(0); } - - //! Entry notification - /** Invoked from inside observe(true) call and whenever a worker enters the arena - this observer is associated with. If a thread is already in the arena when - the observer is activated, the entry notification is called before it - executes the first stolen task. - - Obsolete semantics. For global observers it is called by a thread before - the first steal since observation became enabled. **/ - virtual void on_scheduler_entry( bool /*is_worker*/ ) {} - - //! Exit notification - /** Invoked from inside observe(false) call and whenever a worker leaves the - arena this observer is associated with. - - Obsolete semantics. For global observers it is called by a thread before - the first steal since observation became enabled. **/ - virtual void on_scheduler_exit( bool /*is_worker*/ ) {} - - //! Destructor automatically switches observation off if it is enabled. - virtual ~task_scheduler_observer_v3() { if(my_proxy) observe(false);} -}; - -} // namespace internal - -#if __TBB_ARENA_OBSERVER -namespace interface6 { -class task_scheduler_observer : public internal::task_scheduler_observer_v3 { - friend class internal::task_scheduler_observer_v3; - friend class internal::observer_proxy; - friend class internal::observer_list; - - /** Negative numbers with the largest absolute value to minimize probability - of coincidence in case of a bug in busy count usage. **/ - // TODO: take more high bits for version number - static const intptr_t v6_trait = (intptr_t)((~(uintptr_t)0 >> 1) + 1); - - //! contains task_arena pointer or tag indicating local or global semantics of the observer - intptr_t my_context_tag; - enum { global_tag = 0, implicit_tag = 1 }; - -public: - //! Construct local or global observer in inactive state (observation disabled). - /** For a local observer entry/exit notifications are invoked whenever a worker - thread joins/leaves the arena of the observer's owner thread. If a thread is - already in the arena when the observer is activated, the entry notification is - called before it executes the first stolen task. **/ - /** TODO: Obsolete. - Global observer semantics is obsolete as it violates master thread isolation - guarantees and is not composable. Thus the current default behavior of the - constructor is obsolete too and will be changed in one of the future versions - of the library. **/ - explicit task_scheduler_observer( bool local = false ) { -#if __TBB_ARENA_OBSERVER - my_context_tag = local? implicit_tag : global_tag; -#else - __TBB_ASSERT_EX( !local, NULL ); - my_context_tag = global_tag; -#endif - } - -#if __TBB_ARENA_OBSERVER - //! Construct local observer for a given arena in inactive state (observation disabled). - /** entry/exit notifications are invoked whenever a thread joins/leaves arena. - If a thread is already in the arena when the observer is activated, the entry notification - is called before it executes the first stolen task. **/ - explicit task_scheduler_observer( task_arena & a) { - my_context_tag = (intptr_t)&a; - } -#endif /* __TBB_ARENA_OBSERVER */ - - /** Destructor protects instance of the observer from concurrent notification. - It is recommended to disable observation before destructor of a derived class starts, - otherwise it can lead to concurrent notification callback on partly destroyed object **/ - virtual ~task_scheduler_observer() { if(my_proxy) observe(false); } - - //! Enable or disable observation - /** Warning: concurrent invocations of this method are not safe. - Repeated calls with the same state are no-ops. **/ - void observe( bool state=true ) { - if( state && !my_proxy ) { - __TBB_ASSERT( !my_busy_count, "Inconsistent state of task_scheduler_observer instance"); - my_busy_count.store(v6_trait); - } - internal::task_scheduler_observer_v3::observe(state); - } -}; - -} //namespace interface6 -using interface6::task_scheduler_observer; -#else /*__TBB_ARENA_OBSERVER*/ -typedef tbb::internal::task_scheduler_observer_v3 task_scheduler_observer; -#endif /*__TBB_ARENA_OBSERVER*/ - -} // namespace tbb - -#endif /* __TBB_SCHEDULER_OBSERVER */ - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_task_scheduler_observer_H_include_area - -#endif /* __TBB_task_scheduler_observer_H */ +#include "../oneapi/tbb/task_scheduler_observer.h" diff --git a/src/3rd party/tbb/tbb.h b/src/3rd party/tbb/tbb.h index d6c2d7d64d..e443b8f1ca 100644 --- a/src/3rd party/tbb/tbb.h +++ b/src/3rd party/tbb/tbb.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,84 +14,4 @@ limitations under the License. */ -#ifndef __TBB_tbb_H -#define __TBB_tbb_H - -#if (!defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) || (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0)) && !defined(__TBB_INTERNAL_INCLUDES_DEPRECATION_MESSAGE) -#pragma message("TBB Warning: tbb.h contains deprecated functionality. For details, please see Deprecated Features appendix in the TBB reference manual.") -#endif - -#define __TBB_tbb_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -/** - This header bulk-includes declarations or definitions of all the functionality - provided by TBB (save for tbbmalloc and 3rd party dependent headers). - - If you use only a few TBB constructs, consider including specific headers only. - Any header listed below can be included independently of others. -**/ - -#if TBB_PREVIEW_AGGREGATOR -#include "aggregator.h" -#endif -#include "aligned_space.h" -#include "atomic.h" -#include "blocked_range.h" -#include "blocked_range2d.h" -#include "blocked_range3d.h" -#if TBB_PREVIEW_BLOCKED_RANGE_ND -#include "blocked_rangeNd.h" -#endif -#include "cache_aligned_allocator.h" -#include "combinable.h" -#include "concurrent_hash_map.h" -#if TBB_PREVIEW_CONCURRENT_LRU_CACHE -#include "concurrent_lru_cache.h" -#endif -#include "concurrent_priority_queue.h" -#include "concurrent_queue.h" -#include "concurrent_unordered_map.h" -#include "concurrent_unordered_set.h" -#if TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS -#include "concurrent_map.h" -#include "concurrent_set.h" -#endif -#include "concurrent_vector.h" -#include "critical_section.h" -#include "enumerable_thread_specific.h" -#include "flow_graph.h" -#include "global_control.h" -#include "iterators.h" -#include "mutex.h" -#include "null_mutex.h" -#include "null_rw_mutex.h" -#include "parallel_do.h" -#include "parallel_for.h" -#include "parallel_for_each.h" -#include "parallel_invoke.h" -#include "parallel_reduce.h" -#include "parallel_scan.h" -#include "parallel_sort.h" -#include "partitioner.h" -#include "pipeline.h" -#include "queuing_mutex.h" -#include "queuing_rw_mutex.h" -#include "reader_writer_lock.h" -#include "recursive_mutex.h" -#include "spin_mutex.h" -#include "spin_rw_mutex.h" -#include "task.h" -#include "task_arena.h" -#include "task_group.h" -#include "task_scheduler_init.h" -#include "task_scheduler_observer.h" -#include "tbb_allocator.h" -#include "tbb_exception.h" -#include "tbb_thread.h" -#include "tick_count.h" - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_tbb_H_include_area - -#endif /* __TBB_tbb_H */ +#include "../oneapi/tbb.h" diff --git a/src/3rd party/tbb/tbb_allocator.h b/src/3rd party/tbb/tbb_allocator.h index 206593870c..81ab9d33b5 100644 --- a/src/3rd party/tbb/tbb_allocator.h +++ b/src/3rd party/tbb/tbb_allocator.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,190 +14,4 @@ limitations under the License. */ -#ifndef __TBB_tbb_allocator_H -#define __TBB_tbb_allocator_H - -#include "tbb_stddef.h" -#include -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - #include // std::forward -#endif -#include - -namespace tbb { - -//! @cond INTERNAL -namespace internal { - - //! Deallocates memory using FreeHandler - /** The function uses scalable_free if scalable allocator is available and free if not*/ - void __TBB_EXPORTED_FUNC deallocate_via_handler_v3( void *p ); - - //! Allocates memory using MallocHandler - /** The function uses scalable_malloc if scalable allocator is available and malloc if not*/ - void* __TBB_EXPORTED_FUNC allocate_via_handler_v3( size_t n ); - - //! Returns true if standard malloc/free are used to work with memory. - bool __TBB_EXPORTED_FUNC is_malloc_used_v3(); -} -//! @endcond - -#if _MSC_VER && !defined(__INTEL_COMPILER) - // Workaround for erroneous "unreferenced parameter" warning in method destroy. - #pragma warning (push) - #pragma warning (disable: 4100) -#endif - -//! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 -/** The class selects the best memory allocation mechanism available - from scalable_malloc and standard malloc. - The members are ordered the same way they are in section 20.4.1 - of the ISO C++ standard. - @ingroup memory_allocation */ -template -class tbb_allocator { -public: - typedef typename internal::allocator_type::value_type value_type; - typedef value_type* pointer; - typedef const value_type* const_pointer; - typedef value_type& reference; - typedef const value_type& const_reference; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - template struct rebind { - typedef tbb_allocator other; - }; - - //! Specifies current allocator - enum malloc_type { - scalable, - standard - }; - - tbb_allocator() throw() {} - tbb_allocator( const tbb_allocator& ) throw() {} - template tbb_allocator(const tbb_allocator&) throw() {} - - pointer address(reference x) const {return &x;} - const_pointer address(const_reference x) const {return &x;} - - //! Allocate space for n objects. - pointer allocate( size_type n, const void* /*hint*/ = 0) { - return pointer(internal::allocate_via_handler_v3( n * sizeof(value_type) )); - } - - //! Free previously allocated block of memory. - void deallocate( pointer p, size_type ) { - internal::deallocate_via_handler_v3(p); - } - - //! Largest value for which method allocate might succeed. - size_type max_size() const throw() { - size_type max = static_cast(-1) / sizeof (value_type); - return (max > 0 ? max : 1); - } - - //! Copy-construct value at location pointed to by p. -#if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - template - void construct(U *p, Args&&... args) - { ::new((void *)p) U(std::forward(args)...); } -#else // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC -#if __TBB_CPP11_RVALUE_REF_PRESENT - void construct( pointer p, value_type&& value ) {::new((void*)(p)) value_type(std::move(value));} -#endif - void construct( pointer p, const value_type& value ) {::new((void*)(p)) value_type(value);} -#endif // __TBB_ALLOCATOR_CONSTRUCT_VARIADIC - - //! Destroy value at location pointed to by p. - void destroy( pointer p ) {p->~value_type();} - - //! Returns current allocator - static malloc_type allocator_type() { - return internal::is_malloc_used_v3() ? standard : scalable; - } -}; - -#if _MSC_VER && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif // warning 4100 is back - -//! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1 -/** @ingroup memory_allocation */ -template<> -class tbb_allocator { -public: - typedef void* pointer; - typedef const void* const_pointer; - typedef void value_type; - template struct rebind { - typedef tbb_allocator other; - }; -}; - -template -inline bool operator==( const tbb_allocator&, const tbb_allocator& ) {return true;} - -template -inline bool operator!=( const tbb_allocator&, const tbb_allocator& ) {return false;} - -//! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5 -/** The class is an adapter over an actual allocator that fills the allocation - using memset function with template argument C as the value. - The members are ordered the same way they are in section 20.4.1 - of the ISO C++ standard. - @ingroup memory_allocation */ -template class Allocator = tbb_allocator> -class zero_allocator : public Allocator -{ -public: - typedef Allocator base_allocator_type; - typedef typename base_allocator_type::value_type value_type; - typedef typename base_allocator_type::pointer pointer; - typedef typename base_allocator_type::const_pointer const_pointer; - typedef typename base_allocator_type::reference reference; - typedef typename base_allocator_type::const_reference const_reference; - typedef typename base_allocator_type::size_type size_type; - typedef typename base_allocator_type::difference_type difference_type; - template struct rebind { - typedef zero_allocator other; - }; - - zero_allocator() throw() { } - zero_allocator(const zero_allocator &a) throw() : base_allocator_type( a ) { } - template - zero_allocator(const zero_allocator &a) throw() : base_allocator_type( Allocator( a ) ) { } - - pointer allocate(const size_type n, const void *hint = 0 ) { - pointer ptr = base_allocator_type::allocate( n, hint ); - std::memset( static_cast(ptr), 0, n * sizeof(value_type) ); - return ptr; - } -}; - -//! Analogous to std::allocator, as defined in ISO C++ Standard, Section 20.4.1 -/** @ingroup memory_allocation */ -template class Allocator> -class zero_allocator : public Allocator { -public: - typedef Allocator base_allocator_type; - typedef typename base_allocator_type::value_type value_type; - typedef typename base_allocator_type::pointer pointer; - typedef typename base_allocator_type::const_pointer const_pointer; - template struct rebind { - typedef zero_allocator other; - }; -}; - -template class B1, typename T2, template class B2> -inline bool operator==( const zero_allocator &a, const zero_allocator &b) { - return static_cast< B1 >(a) == static_cast< B2 >(b); -} -template class B1, typename T2, template class B2> -inline bool operator!=( const zero_allocator &a, const zero_allocator &b) { - return static_cast< B1 >(a) != static_cast< B2 >(b); -} - -} // namespace tbb - -#endif /* __TBB_tbb_allocator_H */ +#include "../oneapi/tbb/tbb_allocator.h" diff --git a/src/3rd party/tbb/tbb_config.h b/src/3rd party/tbb/tbb_config.h deleted file mode 100644 index 7b1f02fdd6..0000000000 --- a/src/3rd party/tbb/tbb_config.h +++ /dev/null @@ -1,873 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_tbb_config_H -#define __TBB_tbb_config_H - -/** This header is supposed to contain macro definitions and C style comments only. - The macros defined here are intended to control such aspects of TBB build as - - presence of compiler features - - compilation modes - - feature sets - - known compiler/platform issues -**/ - -/* This macro marks incomplete code or comments describing ideas which are considered for the future. - * See also for plain comment with TODO and FIXME marks for small improvement opportunities. - */ -#define __TBB_TODO 0 - -/* Check which standard library we use. */ -/* __TBB_SYMBOL is defined only while processing exported symbols list where C++ is not allowed. */ -#if !defined(__TBB_SYMBOL) && !__TBB_CONFIG_PREPROC_ONLY - #include -#endif - -// Note that when ICC or Clang is in use, __TBB_GCC_VERSION might not fully match -// the actual GCC version on the system. -#define __TBB_GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) - -// Prior to GCC 7, GNU libstdc++ did not have a convenient version macro. -// Therefore we use different ways to detect its version. -#ifdef TBB_USE_GLIBCXX_VERSION -// The version is explicitly specified in our public TBB_USE_GLIBCXX_VERSION macro. -// Its format should match the __TBB_GCC_VERSION above, e.g. 70301 for libstdc++ coming with GCC 7.3.1. -#define __TBB_GLIBCXX_VERSION TBB_USE_GLIBCXX_VERSION -#elif _GLIBCXX_RELEASE && _GLIBCXX_RELEASE != __GNUC__ -// Reported versions of GCC and libstdc++ do not match; trust the latter -#define __TBB_GLIBCXX_VERSION (_GLIBCXX_RELEASE*10000) -#elif __GLIBCPP__ || __GLIBCXX__ -// The version macro is not defined or matches the GCC version; use __TBB_GCC_VERSION -#define __TBB_GLIBCXX_VERSION __TBB_GCC_VERSION -#endif - -#if __clang__ - // according to clang documentation, version can be vendor specific - #define __TBB_CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) -#endif - -/** Target OS is either iOS* or iOS* simulator **/ -#if __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ - #define __TBB_IOS 1 -#endif - -#if __APPLE__ - #if __INTEL_COMPILER && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ > 1099 \ - && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101000 - // ICC does not correctly set the macro if -mmacosx-min-version is not specified - #define __TBB_MACOS_TARGET_VERSION (100000 + 10*(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ - 1000)) - #else - #define __TBB_MACOS_TARGET_VERSION __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ - #endif -#endif - -/** Preprocessor symbols to determine HW architecture **/ - -#if _WIN32||_WIN64 -# if defined(_M_X64)||defined(__x86_64__) // the latter for MinGW support -# define __TBB_x86_64 1 -# elif defined(_M_IA64) -# define __TBB_ipf 1 -# elif defined(_M_IX86)||defined(__i386__) // the latter for MinGW support -# define __TBB_x86_32 1 -# else -# define __TBB_generic_arch 1 -# endif -#else /* Assume generic Unix */ -# if !__linux__ && !__APPLE__ -# define __TBB_generic_os 1 -# endif -# if __TBB_IOS -# define __TBB_generic_arch 1 -# elif __x86_64__ -# define __TBB_x86_64 1 -# elif __ia64__ -# define __TBB_ipf 1 -# elif __i386__||__i386 // __i386 is for Sun OS -# define __TBB_x86_32 1 -# else -# define __TBB_generic_arch 1 -# endif -#endif - -#if __MIC__ || __MIC2__ -#define __TBB_DEFINE_MIC 1 -#endif - -#define __TBB_TSX_AVAILABLE ((__TBB_x86_32 || __TBB_x86_64) && !__TBB_DEFINE_MIC) - -/** Presence of compiler features **/ - -#if __INTEL_COMPILER == 9999 && __INTEL_COMPILER_BUILD_DATE == 20110811 -/* Intel(R) Composer XE 2011 Update 6 incorrectly sets __INTEL_COMPILER. Fix it. */ - #undef __INTEL_COMPILER - #define __INTEL_COMPILER 1210 -#endif - -#if __clang__ && !__INTEL_COMPILER -#define __TBB_USE_OPTIONAL_RTTI __has_feature(cxx_rtti) -#elif defined(_CPPRTTI) -#define __TBB_USE_OPTIONAL_RTTI 1 -#else -#define __TBB_USE_OPTIONAL_RTTI (__GXX_RTTI || __RTTI || __INTEL_RTTI__) -#endif - -#if __TBB_GCC_VERSION >= 40400 && !defined(__INTEL_COMPILER) - /** warning suppression pragmas available in GCC since 4.4 **/ - #define __TBB_GCC_WARNING_SUPPRESSION_PRESENT 1 -#endif - -/* Select particular features of C++11 based on compiler version. - ICC 12.1 (Linux*), GCC 4.3 and higher, clang 2.9 and higher - set __GXX_EXPERIMENTAL_CXX0X__ in c++11 mode. - - Compilers that mimics other compilers (ICC, clang) must be processed before - compilers they mimic (GCC, MSVC). - - TODO: The following conditions should be extended when new compilers/runtimes - support added. - */ - -/** - __TBB_CPP11_PRESENT macro indicates that the compiler supports vast majority of C++11 features. - Depending on the compiler, some features might still be unsupported or work incorrectly. - Use it when enabling C++11 features individually is not practical, and be aware that - some "good enough" compilers might be excluded. **/ -#define __TBB_CPP11_PRESENT (__cplusplus >= 201103L || _MSC_VER >= 1900) - -#define __TBB_CPP17_FALLTHROUGH_PRESENT (__cplusplus >= 201703L) -#define __TBB_FALLTHROUGH_PRESENT (__TBB_GCC_VERSION >= 70000 && !__INTEL_COMPILER) - -/** C++11 mode detection macros for Intel(R) C++ Compiler (enabled by -std=c++XY option): - __INTEL_CXX11_MODE__ for version >=13.0 (not available for ICC 15.0 if -std=c++14 is used), - __STDC_HOSTED__ for version >=12.0 (useful only on Windows), - __GXX_EXPERIMENTAL_CXX0X__ for version >=12.0 on Linux and macOS. **/ -#if __INTEL_COMPILER && !__INTEL_CXX11_MODE__ - // __INTEL_CXX11_MODE__ is not set, try to deduce it - #define __INTEL_CXX11_MODE__ (__GXX_EXPERIMENTAL_CXX0X__ || (_MSC_VER && __STDC_HOSTED__)) -#endif - -#if __INTEL_COMPILER && (!_MSC_VER || __INTEL_CXX11_MODE__) - // On Windows, C++11 features supported by Visual Studio 2010 and higher are enabled by default, - // so in absence of /Qstd= use MSVC branch for feature detection. - // On other platforms, no -std= means C++03. - - #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT (__INTEL_CXX11_MODE__ && __VARIADIC_TEMPLATES) - // Both r-value reference support in compiler and std::move/std::forward - // presence in C++ standard library is checked. - #define __TBB_CPP11_RVALUE_REF_PRESENT ((_MSC_VER >= 1700 || __GXX_EXPERIMENTAL_CXX0X__ && (__TBB_GLIBCXX_VERSION >= 40500 || _LIBCPP_VERSION)) && __INTEL_COMPILER >= 1400) - #define __TBB_IMPLICIT_MOVE_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400 && (_MSC_VER >= 1900 || __TBB_GCC_VERSION >= 40600 || __clang__)) - #if _MSC_VER >= 1600 - #define __TBB_EXCEPTION_PTR_PRESENT ( __INTEL_COMPILER > 1300 \ - /*ICC 12.1 Upd 10 and 13 beta Upd 2 fixed exception_ptr linking issue*/ \ - || (__INTEL_COMPILER == 1300 && __INTEL_COMPILER_BUILD_DATE >= 20120530) \ - || (__INTEL_COMPILER == 1210 && __INTEL_COMPILER_BUILD_DATE >= 20120410) ) - /** libstdc++ that comes with GCC 4.6 use C++11 features not supported by ICC 12.1. - * Because of that ICC 12.1 does not support C++11 mode with gcc 4.6 (or higher), - * and therefore does not define __GXX_EXPERIMENTAL_CXX0X__ macro **/ - #elif __TBB_GLIBCXX_VERSION >= 40404 && __TBB_GLIBCXX_VERSION < 40600 - #define __TBB_EXCEPTION_PTR_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1200) - #elif __TBB_GLIBCXX_VERSION >= 40600 - #define __TBB_EXCEPTION_PTR_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1300) - #elif _LIBCPP_VERSION - #define __TBB_EXCEPTION_PTR_PRESENT __GXX_EXPERIMENTAL_CXX0X__ - #else - #define __TBB_EXCEPTION_PTR_PRESENT 0 - #endif - #define __TBB_STATIC_ASSERT_PRESENT (__INTEL_CXX11_MODE__ || _MSC_VER >= 1600) - #define __TBB_CPP11_TUPLE_PRESENT (_MSC_VER >= 1600 || __GXX_EXPERIMENTAL_CXX0X__ && (__TBB_GLIBCXX_VERSION >= 40300 || _LIBCPP_VERSION)) - #define __TBB_INITIALIZER_LISTS_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400 && (_MSC_VER >= 1800 || __TBB_GLIBCXX_VERSION >= 40400 || _LIBCPP_VERSION)) - #define __TBB_CONSTEXPR_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400) - #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1200) - /** ICC seems to disable support of noexcept event in c++11 when compiling in compatibility mode for gcc <4.6 **/ - #define __TBB_NOEXCEPT_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1300 && (__TBB_GLIBCXX_VERSION >= 40600 || _LIBCPP_VERSION || _MSC_VER)) - #define __TBB_CPP11_STD_BEGIN_END_PRESENT (_MSC_VER >= 1700 || __GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1310 && (__TBB_GLIBCXX_VERSION >= 40600 || _LIBCPP_VERSION)) - #define __TBB_CPP11_AUTO_PRESENT (_MSC_VER >= 1600 || __GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1210) - #define __TBB_CPP11_DECLTYPE_PRESENT (_MSC_VER >= 1600 || __GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1210) - #define __TBB_CPP11_LAMBDAS_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1200) - #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT (_MSC_VER >= 1800 || __GXX_EXPERIMENTAL_CXX0X__ && __INTEL_COMPILER >= 1210) - #define __TBB_OVERRIDE_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1400) - #define __TBB_ALIGNAS_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1500) - #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1210) - #define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (__cplusplus >= 201402L) - #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__cplusplus >= 201402L) - #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__INTEL_COMPILER > 1910) // a future version - #define __TBB_CPP17_INVOKE_RESULT_PRESENT (__cplusplus >= 201703L) -#elif __clang__ -/** TODO: these options need to be rechecked **/ - #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT __has_feature(__cxx_variadic_templates__) - #define __TBB_CPP11_RVALUE_REF_PRESENT (__has_feature(__cxx_rvalue_references__) && (_LIBCPP_VERSION || __TBB_GLIBCXX_VERSION >= 40500)) - #define __TBB_IMPLICIT_MOVE_PRESENT __has_feature(cxx_implicit_moves) -/** TODO: extend exception_ptr related conditions to cover libstdc++ **/ - #define __TBB_EXCEPTION_PTR_PRESENT (__cplusplus >= 201103L && (_LIBCPP_VERSION || __TBB_GLIBCXX_VERSION >= 40600)) - #define __TBB_STATIC_ASSERT_PRESENT __has_feature(__cxx_static_assert__) - #if (__cplusplus >= 201103L && __has_include()) - #define __TBB_CPP11_TUPLE_PRESENT 1 - #endif - #if (__has_feature(__cxx_generalized_initializers__) && __has_include()) - #define __TBB_INITIALIZER_LISTS_PRESENT 1 - #endif - #define __TBB_CONSTEXPR_PRESENT __has_feature(__cxx_constexpr__) - #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT (__has_feature(__cxx_defaulted_functions__) && __has_feature(__cxx_deleted_functions__)) - /**For some unknown reason __has_feature(__cxx_noexcept) does not yield true for all cases. Compiler bug ? **/ - #define __TBB_NOEXCEPT_PRESENT (__cplusplus >= 201103L) - #define __TBB_CPP11_STD_BEGIN_END_PRESENT (__has_feature(__cxx_range_for__) && (_LIBCPP_VERSION || __TBB_GLIBCXX_VERSION >= 40600)) - #define __TBB_CPP11_AUTO_PRESENT __has_feature(__cxx_auto_type__) - #define __TBB_CPP11_DECLTYPE_PRESENT __has_feature(__cxx_decltype__) - #define __TBB_CPP11_LAMBDAS_PRESENT __has_feature(cxx_lambdas) - #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT __has_feature(cxx_default_function_template_args) - #define __TBB_OVERRIDE_PRESENT __has_feature(cxx_override_control) - #define __TBB_ALIGNAS_PRESENT __has_feature(cxx_alignas) - #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT __has_feature(cxx_alias_templates) - #define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (__cplusplus >= 201402L) - #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__has_feature(cxx_variable_templates)) - #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__has_feature(__cpp_deduction_guides)) - #define __TBB_CPP17_INVOKE_RESULT_PRESENT (__has_feature(__cpp_lib_is_invocable)) -#elif __GNUC__ - #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT __GXX_EXPERIMENTAL_CXX0X__ - #define __TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40700) - #define __TBB_CPP11_RVALUE_REF_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40500) - #define __TBB_IMPLICIT_MOVE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40600) - /** __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 here is a substitution for _GLIBCXX_ATOMIC_BUILTINS_4, which is a prerequisite - for exception_ptr but cannot be used in this file because it is defined in a header, not by the compiler. - If the compiler has no atomic intrinsics, the C++ library should not expect those as well. **/ - #define __TBB_EXCEPTION_PTR_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40404 && __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) - #define __TBB_STATIC_ASSERT_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40300) - #define __TBB_CPP11_TUPLE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40300) - #define __TBB_INITIALIZER_LISTS_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40400) - /** gcc seems have to support constexpr from 4.4 but tests in (test_atomic) seeming reasonable fail to compile prior 4.6**/ - #define __TBB_CONSTEXPR_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40400) - #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40400) - #define __TBB_NOEXCEPT_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40600) - #define __TBB_CPP11_STD_BEGIN_END_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40600) - #define __TBB_CPP11_AUTO_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40400) - #define __TBB_CPP11_DECLTYPE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40400) - #define __TBB_CPP11_LAMBDAS_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40500) - #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40300) - #define __TBB_OVERRIDE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40700) - #define __TBB_ALIGNAS_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40800) - #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GCC_VERSION >= 40700) - #define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (__cplusplus >= 201402L && __TBB_GCC_VERSION >= 50000) - #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__cplusplus >= 201402L && __TBB_GCC_VERSION >= 50000) - #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__cpp_deduction_guides >= 201606L) - #define __TBB_CPP17_INVOKE_RESULT_PRESENT (__cplusplus >= 201703L && __TBB_GCC_VERSION >= 70000) -#elif _MSC_VER - // These definitions are also used with Intel C++ Compiler in "default" mode (__INTEL_CXX11_MODE__ == 0); - // see a comment in "__INTEL_COMPILER" section above. - - #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT (_MSC_VER >= 1800) - // Contains a workaround for ICC 13 - #define __TBB_CPP11_RVALUE_REF_PRESENT (_MSC_VER >= 1700 && (!__INTEL_COMPILER || __INTEL_COMPILER >= 1400)) - #define __TBB_IMPLICIT_MOVE_PRESENT (_MSC_VER >= 1900) - #define __TBB_EXCEPTION_PTR_PRESENT (_MSC_VER >= 1600) - #define __TBB_STATIC_ASSERT_PRESENT (_MSC_VER >= 1600) - #define __TBB_CPP11_TUPLE_PRESENT (_MSC_VER >= 1600) - #define __TBB_INITIALIZER_LISTS_PRESENT (_MSC_VER >= 1800) - #define __TBB_CONSTEXPR_PRESENT (_MSC_VER >= 1900) - #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT (_MSC_VER >= 1800) - #define __TBB_NOEXCEPT_PRESENT (_MSC_VER >= 1900) - #define __TBB_CPP11_STD_BEGIN_END_PRESENT (_MSC_VER >= 1700) - #define __TBB_CPP11_AUTO_PRESENT (_MSC_VER >= 1600) - #define __TBB_CPP11_DECLTYPE_PRESENT (_MSC_VER >= 1600) - #define __TBB_CPP11_LAMBDAS_PRESENT (_MSC_VER >= 1600) - #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT (_MSC_VER >= 1800) - #define __TBB_OVERRIDE_PRESENT (_MSC_VER >= 1700) - #define __TBB_ALIGNAS_PRESENT (_MSC_VER >= 1900) - #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT (_MSC_VER >= 1800) - #define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (_MSC_VER >= 1900) - /* Variable templates are supported in VS2015 Update 2 or later */ - #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (_MSC_FULL_VER >= 190023918 && (!__INTEL_COMPILER || __INTEL_COMPILER >= 1700)) - #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (_MSVC_LANG >= 201703L && _MSC_VER >= 1914) - #define __TBB_CPP17_INVOKE_RESULT_PRESENT (_MSVC_LANG >= 201703L && _MSC_VER >= 1911) -#else - #define __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT __TBB_CPP11_PRESENT - #define __TBB_CPP11_RVALUE_REF_PRESENT __TBB_CPP11_PRESENT - #define __TBB_IMPLICIT_MOVE_PRESENT __TBB_CPP11_PRESENT - #define __TBB_EXCEPTION_PTR_PRESENT __TBB_CPP11_PRESENT - #define __TBB_STATIC_ASSERT_PRESENT __TBB_CPP11_PRESENT - #define __TBB_CPP11_TUPLE_PRESENT __TBB_CPP11_PRESENT - #define __TBB_INITIALIZER_LISTS_PRESENT __TBB_CPP11_PRESENT - #define __TBB_CONSTEXPR_PRESENT __TBB_CPP11_PRESENT - #define __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT __TBB_CPP11_PRESENT - #define __TBB_NOEXCEPT_PRESENT __TBB_CPP11_PRESENT - #define __TBB_CPP11_STD_BEGIN_END_PRESENT __TBB_CPP11_PRESENT - #define __TBB_CPP11_AUTO_PRESENT __TBB_CPP11_PRESENT - #define __TBB_CPP11_DECLTYPE_PRESENT __TBB_CPP11_PRESENT - #define __TBB_CPP11_LAMBDAS_PRESENT __TBB_CPP11_PRESENT - #define __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT __TBB_CPP11_PRESENT - #define __TBB_OVERRIDE_PRESENT __TBB_CPP11_PRESENT - #define __TBB_ALIGNAS_PRESENT __TBB_CPP11_PRESENT - #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT __TBB_CPP11_PRESENT - #define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (__cplusplus >= 201402L) - #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__cplusplus >= 201402L) - #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__cplusplus >= 201703L) - #define __TBB_CPP17_INVOKE_RESULT_PRESENT (__cplusplus >= 201703L) -#endif - -// C++11 standard library features - -#define __TBB_CPP11_ARRAY_PRESENT (_MSC_VER >= 1700 || _LIBCPP_VERSION || __GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40300) - -#ifndef __TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT -#define __TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#endif -#define __TBB_CPP11_VARIADIC_TUPLE_PRESENT (!_MSC_VER || _MSC_VER >= 1800) - -#define __TBB_CPP11_TYPE_PROPERTIES_PRESENT (_LIBCPP_VERSION || _MSC_VER >= 1700 || (__TBB_GLIBCXX_VERSION >= 50000 && __GXX_EXPERIMENTAL_CXX0X__)) -// GCC supported some of type properties since 4.7 -#define __TBB_CPP11_IS_COPY_CONSTRUCTIBLE_PRESENT (__GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40700 || __TBB_CPP11_TYPE_PROPERTIES_PRESENT) - -// In GCC, std::move_if_noexcept appeared later than noexcept -#define __TBB_MOVE_IF_NOEXCEPT_PRESENT (__TBB_NOEXCEPT_PRESENT && (__TBB_GLIBCXX_VERSION >= 40700 || _MSC_VER >= 1900 || _LIBCPP_VERSION)) -#define __TBB_ALLOCATOR_TRAITS_PRESENT (__cplusplus >= 201103L && _LIBCPP_VERSION || _MSC_VER >= 1800 || \ - __GXX_EXPERIMENTAL_CXX0X__ && __TBB_GLIBCXX_VERSION >= 40700 && !(__TBB_GLIBCXX_VERSION == 40700 && __TBB_DEFINE_MIC)) -#define __TBB_MAKE_EXCEPTION_PTR_PRESENT (__TBB_EXCEPTION_PTR_PRESENT && (_MSC_VER >= 1700 || __TBB_GLIBCXX_VERSION >= 40600 || _LIBCPP_VERSION || __SUNPRO_CC)) - -// Due to libc++ limitations in C++03 mode, do not pass rvalues to std::make_shared() -#define __TBB_CPP11_SMART_POINTERS_PRESENT ( _MSC_VER >= 1600 || _LIBCPP_VERSION \ - || ((__cplusplus >= 201103L || __GXX_EXPERIMENTAL_CXX0X__) \ - && (__TBB_GLIBCXX_VERSION >= 40500 || __TBB_GLIBCXX_VERSION >= 40400 && __TBB_USE_OPTIONAL_RTTI)) ) - -#define __TBB_CPP11_FUTURE_PRESENT (_MSC_VER >= 1700 || __TBB_GLIBCXX_VERSION >= 40600 && __GXX_EXPERIMENTAL_CXX0X__ || _LIBCPP_VERSION) - -#define __TBB_CPP11_GET_NEW_HANDLER_PRESENT (_MSC_VER >= 1900 || __TBB_GLIBCXX_VERSION >= 40900 && __GXX_EXPERIMENTAL_CXX0X__ || _LIBCPP_VERSION) - -#define __TBB_CPP17_UNCAUGHT_EXCEPTIONS_PRESENT (_MSC_VER >= 1900 || __GLIBCXX__ && __cpp_lib_uncaught_exceptions \ - || _LIBCPP_VERSION >= 3700 && (!__TBB_MACOS_TARGET_VERSION || __TBB_MACOS_TARGET_VERSION >= 101200)) -// TODO: wait when memory_resource will be fully supported in clang and define the right macro -// Currently it is in experimental stage since 6 version. -#define __TBB_CPP17_MEMORY_RESOURCE_PRESENT (_MSC_VER >= 1913 && (_MSVC_LANG > 201402L || __cplusplus > 201402L) || \ - __GLIBCXX__ && __cpp_lib_memory_resource >= 201603) -#define __TBB_CPP17_HW_INTERFERENCE_SIZE_PRESENT (_MSC_VER >= 1911) -// std::swap is in only since C++11, though MSVC had it at least since VS2005 -#if _MSC_VER>=1400 || _LIBCPP_VERSION || __GXX_EXPERIMENTAL_CXX0X__ -#define __TBB_STD_SWAP_HEADER -#else -#define __TBB_STD_SWAP_HEADER -#endif - -//TODO: not clear how exactly this macro affects exception_ptr - investigate -// On linux ICC fails to find existing std::exception_ptr in libstdc++ without this define -#if __INTEL_COMPILER && __GNUC__ && __TBB_EXCEPTION_PTR_PRESENT && !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) - #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 1 -#endif - -// Work around a bug in MinGW32 -#if __MINGW32__ && __TBB_EXCEPTION_PTR_PRESENT && !defined(_GLIBCXX_ATOMIC_BUILTINS_4) - #define _GLIBCXX_ATOMIC_BUILTINS_4 -#endif - -#if __GNUC__ || __SUNPRO_CC || __IBMCPP__ - /* ICC defines __GNUC__ and so is covered */ - #define __TBB_ATTRIBUTE_ALIGNED_PRESENT 1 -#elif _MSC_VER && (_MSC_VER >= 1300 || __INTEL_COMPILER) - #define __TBB_DECLSPEC_ALIGN_PRESENT 1 -#endif - -/* Actually ICC supports gcc __sync_* intrinsics starting 11.1, - * but 64 bit support for 32 bit target comes in later ones*/ -/* TODO: change the version back to 4.1.2 once macro __TBB_WORD_SIZE become optional */ -/* Assumed that all clang versions have these gcc compatible intrinsics. */ -#if __TBB_GCC_VERSION >= 40306 || __INTEL_COMPILER >= 1200 || __clang__ - /** built-in atomics available in GCC since 4.1.2 **/ - #define __TBB_GCC_BUILTIN_ATOMICS_PRESENT 1 -#endif - -#if __TBB_GCC_VERSION >= 70000 && !__INTEL_COMPILER && !__clang__ - // After GCC7 there was possible reordering problem in generic atomic load/store operations. - // So always using builtins. - #define TBB_USE_GCC_BUILTINS 1 -#endif - -#if __INTEL_COMPILER >= 1200 - /** built-in C++11 style atomics available in ICC since 12.0 **/ - #define __TBB_ICC_BUILTIN_ATOMICS_PRESENT 1 -#endif - -#if _MSC_VER>=1600 && (!__INTEL_COMPILER || __INTEL_COMPILER>=1310) - #define __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT 1 -#endif - -#define __TBB_TSX_INTRINSICS_PRESENT ((__RTM__ || _MSC_VER>=1700 || __INTEL_COMPILER>=1300) && !__TBB_DEFINE_MIC && !__ANDROID__) - -/** Macro helpers **/ -#define __TBB_CONCAT_AUX(A,B) A##B -// The additional level of indirection is needed to expand macros A and B (not to get the AB macro). -// See [cpp.subst] and [cpp.concat] for more details. -#define __TBB_CONCAT(A,B) __TBB_CONCAT_AUX(A,B) -// The IGNORED argument and comma are needed to always have 2 arguments (even when A is empty). -#define __TBB_IS_MACRO_EMPTY(A,IGNORED) __TBB_CONCAT_AUX(__TBB_MACRO_EMPTY,A) -#define __TBB_MACRO_EMPTY 1 - -/** User controlled TBB features & modes **/ -#ifndef TBB_USE_DEBUG -/* -There are four cases that are supported: - 1. "_DEBUG is undefined" means "no debug"; - 2. "_DEBUG defined to something that is evaluated to 0" (including "garbage", as per [cpp.cond]) means "no debug"; - 3. "_DEBUG defined to something that is evaluated to a non-zero value" means "debug"; - 4. "_DEBUG defined to nothing (empty)" means "debug". -*/ -#ifdef _DEBUG -// Check if _DEBUG is empty. -#define __TBB_IS__DEBUG_EMPTY (__TBB_IS_MACRO_EMPTY(_DEBUG,IGNORED)==__TBB_MACRO_EMPTY) -#if __TBB_IS__DEBUG_EMPTY -#define TBB_USE_DEBUG 1 -#else -#define TBB_USE_DEBUG _DEBUG -#endif /* __TBB_IS__DEBUG_EMPTY */ -#else -#define TBB_USE_DEBUG 0 -#endif -#endif /* TBB_USE_DEBUG */ - -#ifndef TBB_USE_ASSERT -#define TBB_USE_ASSERT TBB_USE_DEBUG -#endif /* TBB_USE_ASSERT */ - -#ifndef TBB_USE_THREADING_TOOLS -#define TBB_USE_THREADING_TOOLS TBB_USE_DEBUG -#endif /* TBB_USE_THREADING_TOOLS */ - -#ifndef TBB_USE_PERFORMANCE_WARNINGS -#ifdef TBB_PERFORMANCE_WARNINGS -#define TBB_USE_PERFORMANCE_WARNINGS TBB_PERFORMANCE_WARNINGS -#else -#define TBB_USE_PERFORMANCE_WARNINGS TBB_USE_DEBUG -#endif /* TBB_PERFORMANCE_WARNINGS */ -#endif /* TBB_USE_PERFORMANCE_WARNINGS */ - -#if __TBB_DEFINE_MIC - #if TBB_USE_EXCEPTIONS - #error The platform does not properly support exception handling. Please do not set TBB_USE_EXCEPTIONS macro or set it to 0. - #elif !defined(TBB_USE_EXCEPTIONS) - #define TBB_USE_EXCEPTIONS 0 - #endif -#elif !(__EXCEPTIONS || defined(_CPPUNWIND) || __SUNPRO_CC) - #if TBB_USE_EXCEPTIONS - #error Compilation settings do not support exception handling. Please do not set TBB_USE_EXCEPTIONS macro or set it to 0. - #elif !defined(TBB_USE_EXCEPTIONS) - #define TBB_USE_EXCEPTIONS 0 - #endif -#elif !defined(TBB_USE_EXCEPTIONS) - #define TBB_USE_EXCEPTIONS 1 -#endif - -#ifndef TBB_IMPLEMENT_CPP0X -/** By default, use C++11 classes if available **/ - #if __clang__ - /* Old versions of Intel C++ Compiler do not have __has_include or cannot use it in #define */ - #if (__INTEL_COMPILER && (__INTEL_COMPILER < 1500 || __INTEL_COMPILER == 1500 && __INTEL_COMPILER_UPDATE <= 1)) - #define TBB_IMPLEMENT_CPP0X (__cplusplus < 201103L || !_LIBCPP_VERSION) - #else - #define TBB_IMPLEMENT_CPP0X (__cplusplus < 201103L || (!__has_include() && !__has_include())) - #endif - #elif __GNUC__ - #define TBB_IMPLEMENT_CPP0X (__TBB_GCC_VERSION < 40400 || !__GXX_EXPERIMENTAL_CXX0X__) - #elif _MSC_VER - #define TBB_IMPLEMENT_CPP0X (_MSC_VER < 1700) - #else - // TODO: Reconsider general approach to be more reliable, e.g. (!(__cplusplus >= 201103L && __ STDC_HOSTED__)) - #define TBB_IMPLEMENT_CPP0X (!__STDCPP_THREADS__) - #endif -#endif /* TBB_IMPLEMENT_CPP0X */ - -/* TBB_USE_CAPTURED_EXCEPTION should be explicitly set to either 0 or 1, as it is used as C++ const */ -#ifndef TBB_USE_CAPTURED_EXCEPTION - /** IA-64 architecture pre-built TBB binaries do not support exception_ptr. **/ - #if __TBB_EXCEPTION_PTR_PRESENT && !defined(__ia64__) - #define TBB_USE_CAPTURED_EXCEPTION 0 - #else - #define TBB_USE_CAPTURED_EXCEPTION 1 - #endif -#else /* defined TBB_USE_CAPTURED_EXCEPTION */ - #if !TBB_USE_CAPTURED_EXCEPTION && !__TBB_EXCEPTION_PTR_PRESENT - #error Current runtime does not support std::exception_ptr. Set TBB_USE_CAPTURED_EXCEPTION and make sure that your code is ready to catch tbb::captured_exception. - #endif -#endif /* defined TBB_USE_CAPTURED_EXCEPTION */ - -/** Check whether the request to use GCC atomics can be satisfied **/ -#if TBB_USE_GCC_BUILTINS && !__TBB_GCC_BUILTIN_ATOMICS_PRESENT - #error "GCC atomic built-ins are not supported." -#endif - -/** Internal TBB features & modes **/ - -/** __TBB_CONCURRENT_ORDERED_CONTAINERS indicates that all conditions of use - * concurrent_map and concurrent_set are met. **/ -// TODO: Add cpp11 random generation macro -#ifndef __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT - #define __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT ( __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT \ - && __TBB_IMPLICIT_MOVE_PRESENT && __TBB_CPP11_AUTO_PRESENT && __TBB_CPP11_LAMBDAS_PRESENT && __TBB_CPP11_ARRAY_PRESENT \ - && __TBB_INITIALIZER_LISTS_PRESENT ) -#endif - -/** __TBB_WEAK_SYMBOLS_PRESENT denotes that the system supports the weak symbol mechanism **/ -#ifndef __TBB_WEAK_SYMBOLS_PRESENT -#define __TBB_WEAK_SYMBOLS_PRESENT ( !_WIN32 && !__APPLE__ && !__sun && (__TBB_GCC_VERSION >= 40000 || __INTEL_COMPILER ) ) -#endif - -/** __TBB_DYNAMIC_LOAD_ENABLED describes the system possibility to load shared libraries at run time **/ -#ifndef __TBB_DYNAMIC_LOAD_ENABLED - #define __TBB_DYNAMIC_LOAD_ENABLED 1 -#endif - -/** __TBB_SOURCE_DIRECTLY_INCLUDED is a mode used in whitebox testing when - it's necessary to test internal functions not exported from TBB DLLs -**/ -#if (_WIN32||_WIN64) && (__TBB_SOURCE_DIRECTLY_INCLUDED || TBB_USE_PREVIEW_BINARY) - #define __TBB_NO_IMPLICIT_LINKAGE 1 - #define __TBBMALLOC_NO_IMPLICIT_LINKAGE 1 -#endif - -#ifndef __TBB_COUNT_TASK_NODES - #define __TBB_COUNT_TASK_NODES TBB_USE_ASSERT -#endif - -#ifndef __TBB_TASK_GROUP_CONTEXT - #define __TBB_TASK_GROUP_CONTEXT 1 -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -#ifndef __TBB_SCHEDULER_OBSERVER - #define __TBB_SCHEDULER_OBSERVER 1 -#endif /* __TBB_SCHEDULER_OBSERVER */ - -#ifndef __TBB_FP_CONTEXT - #define __TBB_FP_CONTEXT __TBB_TASK_GROUP_CONTEXT -#endif /* __TBB_FP_CONTEXT */ - -#if __TBB_FP_CONTEXT && !__TBB_TASK_GROUP_CONTEXT - #error __TBB_FP_CONTEXT requires __TBB_TASK_GROUP_CONTEXT to be enabled -#endif - -#define __TBB_RECYCLE_TO_ENQUEUE __TBB_BUILD // keep non-official - -#ifndef __TBB_ARENA_OBSERVER - #define __TBB_ARENA_OBSERVER __TBB_SCHEDULER_OBSERVER -#endif /* __TBB_ARENA_OBSERVER */ - -#ifndef __TBB_TASK_ISOLATION - #define __TBB_TASK_ISOLATION 1 -#endif /* __TBB_TASK_ISOLATION */ - -#if TBB_USE_EXCEPTIONS && !__TBB_TASK_GROUP_CONTEXT - #error TBB_USE_EXCEPTIONS requires __TBB_TASK_GROUP_CONTEXT to be enabled -#endif - -#ifndef __TBB_TASK_PRIORITY - #define __TBB_TASK_PRIORITY (__TBB_TASK_GROUP_CONTEXT) -#endif /* __TBB_TASK_PRIORITY */ - -#if __TBB_TASK_PRIORITY && !__TBB_TASK_GROUP_CONTEXT - #error __TBB_TASK_PRIORITY requires __TBB_TASK_GROUP_CONTEXT to be enabled -#endif - -#if TBB_PREVIEW_NUMA_SUPPORT || __TBB_BUILD - #define __TBB_NUMA_SUPPORT 1 -#endif - -#if TBB_PREVIEW_WAITING_FOR_WORKERS || __TBB_BUILD - #define __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE 1 -#endif - -#ifndef __TBB_ENQUEUE_ENFORCED_CONCURRENCY - #define __TBB_ENQUEUE_ENFORCED_CONCURRENCY 1 -#endif - -#if !defined(__TBB_SURVIVE_THREAD_SWITCH) && \ - (_WIN32 || _WIN64 || __APPLE__ || (__linux__ && !__ANDROID__)) - #define __TBB_SURVIVE_THREAD_SWITCH 1 -#endif /* __TBB_SURVIVE_THREAD_SWITCH */ - -#ifndef __TBB_DEFAULT_PARTITIONER -#define __TBB_DEFAULT_PARTITIONER tbb::auto_partitioner -#endif - -#ifndef __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES -#define __TBB_USE_PROPORTIONAL_SPLIT_IN_BLOCKED_RANGES 1 -#endif - -#ifndef __TBB_ENABLE_RANGE_FEEDBACK -#define __TBB_ENABLE_RANGE_FEEDBACK 0 -#endif - -#ifdef _VARIADIC_MAX - #define __TBB_VARIADIC_MAX _VARIADIC_MAX -#else - #if _MSC_VER == 1700 - #define __TBB_VARIADIC_MAX 5 // VS11 setting, issue resolved in VS12 - #elif _MSC_VER == 1600 - #define __TBB_VARIADIC_MAX 10 // VS10 setting - #else - #define __TBB_VARIADIC_MAX 15 - #endif -#endif - -// Intel C++ Compiler starts analyzing usages of the deprecated content at the template -// instantiation site, which is too late for suppression of the corresponding messages for internal -// stuff. -#if !defined(__INTEL_COMPILER) && (!defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) || (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0)) - #if (__cplusplus >= 201402L) - #define __TBB_DEPRECATED [[deprecated]] - #define __TBB_DEPRECATED_MSG(msg) [[deprecated(msg)]] - #elif _MSC_VER - #define __TBB_DEPRECATED __declspec(deprecated) - #define __TBB_DEPRECATED_MSG(msg) __declspec(deprecated(msg)) - #elif (__GNUC__ && __TBB_GCC_VERSION >= 40805) || __clang__ - #define __TBB_DEPRECATED __attribute__((deprecated)) - #define __TBB_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) - #endif -#endif // !defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) || (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0) - -#if !defined(__TBB_DEPRECATED) - #define __TBB_DEPRECATED - #define __TBB_DEPRECATED_MSG(msg) -#elif !defined(__TBB_SUPPRESS_INTERNAL_DEPRECATED_MESSAGES) - // Suppress deprecated messages from self - #define __TBB_SUPPRESS_INTERNAL_DEPRECATED_MESSAGES 1 -#endif - -#if defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) && (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0) - #define __TBB_DEPRECATED_VERBOSE __TBB_DEPRECATED - #define __TBB_DEPRECATED_VERBOSE_MSG(msg) __TBB_DEPRECATED_MSG(msg) -#else - #define __TBB_DEPRECATED_VERBOSE - #define __TBB_DEPRECATED_VERBOSE_MSG(msg) -#endif // (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0) - -#if (!defined(TBB_SUPPRESS_DEPRECATED_MESSAGES) || (TBB_SUPPRESS_DEPRECATED_MESSAGES == 0)) && !__TBB_CPP11_PRESENT - #pragma message("TBB Warning: Support for C++98/03 is deprecated. Please use the compiler that supports C++11 features at least.") -#endif - -/** __TBB_WIN8UI_SUPPORT enables support of Windows* Store Apps and limit a possibility to load - shared libraries at run time only from application container **/ -// TODO: Separate this single macro into two for Windows 8 Store* (win8ui mode) and UWP/UWD modes. -#if defined(WINAPI_FAMILY) && WINAPI_FAMILY == WINAPI_FAMILY_APP - #define __TBB_WIN8UI_SUPPORT 1 -#else - #define __TBB_WIN8UI_SUPPORT 0 -#endif - -/** Macros of the form __TBB_XXX_BROKEN denote known issues that are caused by - the bugs in compilers, standard or OS specific libraries. They should be - removed as soon as the corresponding bugs are fixed or the buggy OS/compiler - versions go out of the support list. -**/ - -#if __SIZEOF_POINTER__ < 8 && __ANDROID__ && __TBB_GCC_VERSION <= 40403 && !__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 - /** Necessary because on Android 8-byte CAS and F&A are not available for some processor architectures, - but no mandatory warning message appears from GCC 4.4.3. Instead, only a linkage error occurs when - these atomic operations are used (such as in unit test test_atomic.exe). **/ - #define __TBB_GCC_64BIT_ATOMIC_BUILTINS_BROKEN 1 -#elif __TBB_x86_32 && __TBB_GCC_VERSION == 40102 && ! __GNUC_RH_RELEASE__ - /** GCC 4.1.2 erroneously emit call to external function for 64 bit sync_ intrinsics. - However these functions are not defined anywhere. It seems that this problem was fixed later on - and RHEL got an updated version of gcc 4.1.2. **/ - #define __TBB_GCC_64BIT_ATOMIC_BUILTINS_BROKEN 1 -#endif - -#if __GNUC__ && __TBB_x86_64 && __INTEL_COMPILER == 1200 - #define __TBB_ICC_12_0_INL_ASM_FSTCW_BROKEN 1 -#endif - -#if _MSC_VER && __INTEL_COMPILER && (__INTEL_COMPILER<1110 || __INTEL_COMPILER==1110 && __INTEL_COMPILER_BUILD_DATE < 20091012) - /** Necessary to avoid ICL error (or warning in non-strict mode): - "exception specification for implicitly declared virtual destructor is - incompatible with that of overridden one". **/ - #define __TBB_DEFAULT_DTOR_THROW_SPEC_BROKEN 1 -#endif - -#if !__INTEL_COMPILER && (_MSC_VER && _MSC_VER < 1500 || __GNUC__ && __TBB_GCC_VERSION < 40102) - /** gcc 3.4.6 (and earlier) and VS2005 (and earlier) do not allow declaring template class as a friend - of classes defined in other namespaces. **/ - #define __TBB_TEMPLATE_FRIENDS_BROKEN 1 -#endif - -#if __GLIBC__==2 && __GLIBC_MINOR__==3 || (__APPLE__ && ( __INTEL_COMPILER==1200 && !TBB_USE_DEBUG)) - /** Macro controlling EH usages in TBB tests. - Some older versions of glibc crash when exception handling happens concurrently. **/ - #define __TBB_THROW_ACROSS_MODULE_BOUNDARY_BROKEN 1 -#endif - -#if (_WIN32||_WIN64) && __INTEL_COMPILER == 1110 - /** That's a bug in Intel C++ Compiler 11.1.044/IA-32 architecture/Windows* OS, that leads to a worker thread crash on the thread's startup. **/ - #define __TBB_ICL_11_1_CODE_GEN_BROKEN 1 -#endif - -#if __clang__ || (__GNUC__==3 && __GNUC_MINOR__==3 && !defined(__INTEL_COMPILER)) - /** Bugs with access to nested classes declared in protected area */ - #define __TBB_PROTECTED_NESTED_CLASS_BROKEN 1 -#endif - -#if __MINGW32__ && __TBB_GCC_VERSION < 40200 - /** MinGW has a bug with stack alignment for routines invoked from MS RTLs. - Since GCC 4.2, the bug can be worked around via a special attribute. **/ - #define __TBB_SSE_STACK_ALIGNMENT_BROKEN 1 -#endif - -#if __TBB_GCC_VERSION==40300 && !__INTEL_COMPILER && !__clang__ - /* GCC of this version may rashly ignore control dependencies */ - #define __TBB_GCC_OPTIMIZER_ORDERING_BROKEN 1 -#endif - -#if __FreeBSD__ - /** A bug in FreeBSD 8.0 results in kernel panic when there is contention - on a mutex created with this attribute. **/ - #define __TBB_PRIO_INHERIT_BROKEN 1 - - /** A bug in FreeBSD 8.0 results in test hanging when an exception occurs - during (concurrent?) object construction by means of placement new operator. **/ - #define __TBB_PLACEMENT_NEW_EXCEPTION_SAFETY_BROKEN 1 -#endif /* __FreeBSD__ */ - -#if (__linux__ || __APPLE__) && __i386__ && defined(__INTEL_COMPILER) - /** The Intel C++ Compiler for IA-32 architecture (Linux* OS|macOS) crashes or generates - incorrect code when __asm__ arguments have a cast to volatile. **/ - #define __TBB_ICC_ASM_VOLATILE_BROKEN 1 -#endif - -#if !__INTEL_COMPILER && (_MSC_VER && _MSC_VER < 1700 || __GNUC__==3 && __GNUC_MINOR__<=2) - /** Bug in GCC 3.2 and MSVC compilers that sometimes return 0 for __alignof(T) - when T has not yet been instantiated. **/ - #define __TBB_ALIGNOF_NOT_INSTANTIATED_TYPES_BROKEN 1 -#endif - -#if __TBB_DEFINE_MIC - /** Main thread and user's thread have different default thread affinity masks. **/ - #define __TBB_MAIN_THREAD_AFFINITY_BROKEN 1 -#endif - -#if __GXX_EXPERIMENTAL_CXX0X__ && !defined(__EXCEPTIONS) && \ - ((!__INTEL_COMPILER && !__clang__ && (__TBB_GCC_VERSION>=40400 && __TBB_GCC_VERSION<40600)) || \ - (__INTEL_COMPILER<=1400 && (__TBB_GLIBCXX_VERSION>=40400 && __TBB_GLIBCXX_VERSION<=40801))) -/* There is an issue for specific GCC toolchain when C++11 is enabled - and exceptions are disabled: - exceprion_ptr.h/nested_exception.h use throw unconditionally. - GCC can ignore 'throw' since 4.6; but with ICC the issue still exists. - */ - #define __TBB_LIBSTDCPP_EXCEPTION_HEADERS_BROKEN 1 -#endif - -#if __INTEL_COMPILER==1300 && __TBB_GLIBCXX_VERSION>=40700 && defined(__GXX_EXPERIMENTAL_CXX0X__) -/* Some C++11 features used inside libstdc++ are not supported by Intel C++ Compiler. */ - #define __TBB_ICC_13_0_CPP11_STDLIB_SUPPORT_BROKEN 1 -#endif - -#if (__GNUC__==4 && __GNUC_MINOR__==4 ) && !defined(__INTEL_COMPILER) && !defined(__clang__) - /** excessive warnings related to strict aliasing rules in GCC 4.4 **/ - #define __TBB_GCC_STRICT_ALIASING_BROKEN 1 - /* topical remedy: #pragma GCC diagnostic ignored "-Wstrict-aliasing" */ - #if !__TBB_GCC_WARNING_SUPPRESSION_PRESENT - #error Warning suppression is not supported, while should. - #endif -#endif - -/* In a PIC mode some versions of GCC 4.1.2 generate incorrect inlined code for 8 byte __sync_val_compare_and_swap intrinsic */ -#if __TBB_GCC_VERSION == 40102 && __PIC__ && !defined(__INTEL_COMPILER) && !defined(__clang__) - #define __TBB_GCC_CAS8_BUILTIN_INLINING_BROKEN 1 -#endif - -#if __TBB_x86_32 && ( __INTEL_COMPILER || (__GNUC__==5 && __GNUC_MINOR__>=2 && __GXX_EXPERIMENTAL_CXX0X__) \ - || (__GNUC__==3 && __GNUC_MINOR__==3) || (__MINGW32__ && __GNUC__==4 && __GNUC_MINOR__==5) || __SUNPRO_CC ) - // Some compilers for IA-32 architecture fail to provide 8-byte alignment of objects on the stack, - // even if the object specifies 8-byte alignment. On such platforms, the implementation - // of 64 bit atomics for IA-32 architecture (e.g. atomic) use different tactics - // depending upon whether the object is properly aligned or not. - #define __TBB_FORCE_64BIT_ALIGNMENT_BROKEN 1 -#else - // Define to 0 explicitly because the macro is used in a compiled code of test_atomic - #define __TBB_FORCE_64BIT_ALIGNMENT_BROKEN 0 -#endif - -#if __GNUC__ && !__INTEL_COMPILER && !__clang__ && __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT && __TBB_GCC_VERSION < 40700 - #define __TBB_ZERO_INIT_WITH_DEFAULTED_CTOR_BROKEN 1 -#endif - -#if _MSC_VER && _MSC_VER <= 1800 && !__INTEL_COMPILER - // With MSVC, when an array is passed by const reference to a template function, - // constness from the function parameter may get propagated to the template parameter. - #define __TBB_CONST_REF_TO_ARRAY_TEMPLATE_PARAM_BROKEN 1 -#endif - -// A compiler bug: a disabled copy constructor prevents use of the moving constructor -#define __TBB_IF_NO_COPY_CTOR_MOVE_SEMANTICS_BROKEN (_MSC_VER && (__INTEL_COMPILER >= 1300 && __INTEL_COMPILER <= 1310) && !__INTEL_CXX11_MODE__) - -#define __TBB_CPP11_DECLVAL_BROKEN (_MSC_VER == 1600 || (__GNUC__ && __TBB_GCC_VERSION < 40500) ) -// Intel C++ Compiler has difficulties with copying std::pair with VC11 std::reference_wrapper being a const member -#define __TBB_COPY_FROM_NON_CONST_REF_BROKEN (_MSC_VER == 1700 && __INTEL_COMPILER && __INTEL_COMPILER < 1600) - -// The implicit upcasting of the tuple of a reference of a derived class to a base class fails on icc 13.X if the system's gcc environment is 4.8 -// Also in gcc 4.4 standard library the implementation of the tuple<&> conversion (tuple a = tuple, B is inherited from A) is broken. -#if __GXX_EXPERIMENTAL_CXX0X__ && __GLIBCXX__ && ((__INTEL_COMPILER >=1300 && __INTEL_COMPILER <=1310 && __TBB_GLIBCXX_VERSION>=40700) || (__TBB_GLIBCXX_VERSION < 40500)) -#define __TBB_UPCAST_OF_TUPLE_OF_REF_BROKEN 1 -#endif - -// In some cases decltype of a function adds a reference to a return type. -#define __TBB_CPP11_DECLTYPE_OF_FUNCTION_RETURN_TYPE_BROKEN (_MSC_VER == 1600 && !__INTEL_COMPILER) - -// Visual Studio 2013 does not delete the copy constructor when a user-defined move constructor is provided -#if _MSC_VER && _MSC_VER <= 1800 - #define __TBB_IMPLICIT_COPY_DELETION_BROKEN 1 -#endif - -/** End of __TBB_XXX_BROKEN macro section **/ - -#if defined(_MSC_VER) && _MSC_VER>=1500 && !defined(__INTEL_COMPILER) - // A macro to suppress erroneous or benign "unreachable code" MSVC warning (4702) - #define __TBB_MSVC_UNREACHABLE_CODE_IGNORED 1 -#endif - -#define __TBB_ATOMIC_CTORS (__TBB_CONSTEXPR_PRESENT && __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT && (!__TBB_ZERO_INIT_WITH_DEFAULTED_CTOR_BROKEN)) - -// Many OS versions (Android 4.0.[0-3] for example) need workaround for dlopen to avoid non-recursive loader lock hang -// Setting the workaround for all compile targets ($APP_PLATFORM) below Android 4.4 (android-19) -#if __ANDROID__ -#include -#define __TBB_USE_DLOPEN_REENTRANCY_WORKAROUND (__ANDROID_API__ < 19) -#endif - -#define __TBB_ALLOCATOR_CONSTRUCT_VARIADIC (__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT) - -#define __TBB_VARIADIC_PARALLEL_INVOKE (TBB_PREVIEW_VARIADIC_PARALLEL_INVOKE && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT) -#define __TBB_FLOW_GRAPH_CPP11_FEATURES (__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT \ - && __TBB_CPP11_SMART_POINTERS_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_AUTO_PRESENT) \ - && __TBB_CPP11_VARIADIC_TUPLE_PRESENT && __TBB_CPP11_DEFAULT_FUNC_TEMPLATE_ARGS_PRESENT \ - && !__TBB_UPCAST_OF_TUPLE_OF_REF_BROKEN -#define __TBB_PREVIEW_STREAMING_NODE (__TBB_CPP11_VARIADIC_FIXED_LENGTH_EXP_PRESENT && __TBB_FLOW_GRAPH_CPP11_FEATURES \ - && TBB_PREVIEW_FLOW_GRAPH_NODES && !TBB_IMPLEMENT_CPP0X && !__TBB_UPCAST_OF_TUPLE_OF_REF_BROKEN) -#define __TBB_PREVIEW_OPENCL_NODE (__TBB_PREVIEW_STREAMING_NODE && __TBB_CPP11_TEMPLATE_ALIASES_PRESENT) -#define __TBB_PREVIEW_MESSAGE_BASED_KEY_MATCHING (TBB_PREVIEW_FLOW_GRAPH_FEATURES || __TBB_PREVIEW_OPENCL_NODE) -#define __TBB_PREVIEW_ASYNC_MSG (TBB_PREVIEW_FLOW_GRAPH_FEATURES && __TBB_FLOW_GRAPH_CPP11_FEATURES) - - -#ifndef __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES -#define __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES TBB_PREVIEW_FLOW_GRAPH_FEATURES -#endif - -// This feature works only in combination with critical tasks (__TBB_PREVIEW_CRITICAL_TASKS) -#ifndef __TBB_PREVIEW_RESUMABLE_TASKS -#define __TBB_PREVIEW_RESUMABLE_TASKS ((__TBB_CPF_BUILD || TBB_PREVIEW_RESUMABLE_TASKS) && !__TBB_WIN8UI_SUPPORT && !__ANDROID__ && !__TBB_ipf) -#endif - -#ifndef __TBB_PREVIEW_CRITICAL_TASKS -#define __TBB_PREVIEW_CRITICAL_TASKS (__TBB_CPF_BUILD || __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES || __TBB_PREVIEW_RESUMABLE_TASKS) -#endif - -#ifndef __TBB_PREVIEW_FLOW_GRAPH_NODE_SET -#define __TBB_PREVIEW_FLOW_GRAPH_NODE_SET (TBB_PREVIEW_FLOW_GRAPH_FEATURES && __TBB_CPP11_PRESENT && __TBB_FLOW_GRAPH_CPP11_FEATURES) -#endif - -#endif /* __TBB_tbb_config_H */ diff --git a/src/3rd party/tbb/tbb_exception.h b/src/3rd party/tbb/tbb_exception.h deleted file mode 100644 index d9045c474c..0000000000 --- a/src/3rd party/tbb/tbb_exception.h +++ /dev/null @@ -1,362 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_exception_H -#define __TBB_exception_H - -#define __TBB_tbb_exception_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "tbb_stddef.h" -#include -#include // required for bad_alloc definition, operators new -#include // required to construct std exception classes - -namespace tbb { - -//! Exception for concurrent containers -class bad_last_alloc : public std::bad_alloc { -public: - const char* what() const throw() __TBB_override; -#if __TBB_DEFAULT_DTOR_THROW_SPEC_BROKEN - ~bad_last_alloc() throw() __TBB_override {} -#endif -}; - -//! Exception for PPL locks -class __TBB_DEPRECATED improper_lock : public std::exception { -public: - const char* what() const throw() __TBB_override; -}; - -//! Exception for user-initiated abort -class user_abort : public std::exception { -public: - const char* what() const throw() __TBB_override; -}; - -//! Exception for missing wait on structured_task_group -class missing_wait : public std::exception { -public: - const char* what() const throw() __TBB_override; -}; - -//! Exception for repeated scheduling of the same task_handle -class invalid_multiple_scheduling : public std::exception { -public: - const char* what() const throw() __TBB_override; -}; - -namespace internal { -//! Obsolete -void __TBB_EXPORTED_FUNC throw_bad_last_alloc_exception_v4(); - -enum exception_id { - eid_bad_alloc = 1, - eid_bad_last_alloc, - eid_nonpositive_step, - eid_out_of_range, - eid_segment_range_error, - eid_index_range_error, - eid_missing_wait, - eid_invalid_multiple_scheduling, - eid_improper_lock, - eid_possible_deadlock, - eid_operation_not_permitted, - eid_condvar_wait_failed, - eid_invalid_load_factor, - eid_reserved, // free slot for backward compatibility, can be reused. - eid_invalid_swap, - eid_reservation_length_error, - eid_invalid_key, - eid_user_abort, - eid_reserved1, -#if __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE - // This id is used only from inside the library and only for support of CPF functionality. - // So, if we drop the functionality, eid_reserved1 can be safely renamed and reused. - eid_blocking_thread_join_impossible = eid_reserved1, -#endif - eid_bad_tagged_msg_cast, - //! The last enumerator tracks the number of defined IDs. It must remain the last one. - /** When adding new IDs, place them immediately _before_ this comment (that is - _after_ all the existing IDs. NEVER insert new IDs between the existing ones. **/ - eid_max -}; - -//! Gathers all throw operators in one place. -/** Its purpose is to minimize code bloat that can be caused by throw operators - scattered in multiple places, especially in templates. **/ -void __TBB_EXPORTED_FUNC throw_exception_v4 ( exception_id ); - -//! Versionless convenience wrapper for throw_exception_v4() -inline void throw_exception ( exception_id eid ) { throw_exception_v4(eid); } - -} // namespace internal -} // namespace tbb - -#if __TBB_TASK_GROUP_CONTEXT -#include "tbb_allocator.h" -#include //for typeid - -namespace tbb { - -//! Interface to be implemented by all exceptions TBB recognizes and propagates across the threads. -/** If an unhandled exception of the type derived from tbb::tbb_exception is intercepted - by the TBB scheduler in one of the worker threads, it is delivered to and re-thrown in - the root thread. The root thread is the thread that has started the outermost algorithm - or root task sharing the same task_group_context with the guilty algorithm/task (the one - that threw the exception first). - - Note: when documentation mentions workers with respect to exception handling, - masters are implied as well, because they are completely equivalent in this context. - Consequently a root thread can be master or worker thread. - - NOTE: In case of nested algorithms or complex task hierarchies when the nested - levels share (explicitly or by means of implicit inheritance) the task group - context of the outermost level, the exception may be (re-)thrown multiple times - (ultimately - in each worker on each nesting level) before reaching the root - thread at the outermost level. IMPORTANT: if you intercept an exception derived - from this class on a nested level, you must re-throw it in the catch block by means - of the "throw;" operator. - - TBB provides two implementations of this interface: tbb::captured_exception and - template class tbb::movable_exception. See their declarations for more info. **/ -class __TBB_DEPRECATED tbb_exception : public std::exception -{ - /** No operator new is provided because the TBB usage model assumes dynamic - creation of the TBB exception objects only by means of applying move() - operation on an exception thrown out of TBB scheduler. **/ - void* operator new ( size_t ); - -public: -#if __clang__ - // At -O3 or even -O2 optimization level, Clang may fully throw away an empty destructor - // of tbb_exception from destructors of derived classes. As a result, it does not create - // vtable for tbb_exception, which is a required part of TBB binary interface. - // Making the destructor non-empty (with just a semicolon) prevents that optimization. - ~tbb_exception() throw() { /* keep the semicolon! */ ; } -#endif - - //! Creates and returns pointer to the deep copy of this exception object. - /** Move semantics is allowed. **/ - virtual tbb_exception* move() throw() = 0; - - //! Destroys objects created by the move() method. - /** Frees memory and calls destructor for this exception object. - Can and must be used only on objects created by the move method. **/ - virtual void destroy() throw() = 0; - - //! Throws this exception object. - /** Make sure that if you have several levels of derivation from this interface - you implement or override this method on the most derived level. The implementation - is as simple as "throw *this;". Failure to do this will result in exception - of a base class type being thrown. **/ - virtual void throw_self() = 0; - - //! Returns RTTI name of the originally intercepted exception - virtual const char* name() const throw() = 0; - - //! Returns the result of originally intercepted exception's what() method. - virtual const char* what() const throw() __TBB_override = 0; - - /** Operator delete is provided only to allow using existing smart pointers - with TBB exception objects obtained as the result of applying move() - operation on an exception thrown out of TBB scheduler. - - When overriding method move() make sure to override operator delete as well - if memory is allocated not by TBB's scalable allocator. **/ - void operator delete ( void* p ) { - internal::deallocate_via_handler_v3(p); - } -}; - -//! This class is used by TBB to propagate information about unhandled exceptions into the root thread. -/** Exception of this type is thrown by TBB in the root thread (thread that started a parallel - algorithm ) if an unhandled exception was intercepted during the algorithm execution in one - of the workers. - \sa tbb::tbb_exception **/ -class __TBB_DEPRECATED captured_exception : public tbb_exception -{ -public: - captured_exception( const captured_exception& src ) - : tbb_exception(src), my_dynamic(false) - { - set(src.my_exception_name, src.my_exception_info); - } - - captured_exception( const char* name_, const char* info ) - : my_dynamic(false) - { - set(name_, info); - } - - __TBB_EXPORTED_METHOD ~captured_exception() throw(); - - captured_exception& operator= ( const captured_exception& src ) { - if ( this != &src ) { - clear(); - set(src.my_exception_name, src.my_exception_info); - } - return *this; - } - - captured_exception* __TBB_EXPORTED_METHOD move() throw() __TBB_override; - - void __TBB_EXPORTED_METHOD destroy() throw() __TBB_override; - - void throw_self() __TBB_override { __TBB_THROW(*this); } - - const char* __TBB_EXPORTED_METHOD name() const throw() __TBB_override; - - const char* __TBB_EXPORTED_METHOD what() const throw() __TBB_override; - - void __TBB_EXPORTED_METHOD set( const char* name, const char* info ) throw(); - void __TBB_EXPORTED_METHOD clear() throw(); - -private: - //! Used only by method move(). - captured_exception() : my_dynamic(), my_exception_name(), my_exception_info() {} - - //! Functionally equivalent to {captured_exception e(name,info); return e.move();} - static captured_exception* allocate( const char* name, const char* info ); - - bool my_dynamic; - const char* my_exception_name; - const char* my_exception_info; -}; - -//! Template that can be used to implement exception that transfers arbitrary ExceptionData to the root thread -/** Code using TBB can instantiate this template with an arbitrary ExceptionData type - and throw this exception object. Such exceptions are intercepted by the TBB scheduler - and delivered to the root thread (). - \sa tbb::tbb_exception **/ -template -class __TBB_DEPRECATED movable_exception : public tbb_exception -{ - typedef movable_exception self_type; - -public: - movable_exception( const ExceptionData& data_ ) - : my_exception_data(data_) - , my_dynamic(false) - , my_exception_name( -#if TBB_USE_EXCEPTIONS - typeid(self_type).name() -#else /* !TBB_USE_EXCEPTIONS */ - "movable_exception" -#endif /* !TBB_USE_EXCEPTIONS */ - ) - {} - - movable_exception( const movable_exception& src ) throw () - : tbb_exception(src) - , my_exception_data(src.my_exception_data) - , my_dynamic(false) - , my_exception_name(src.my_exception_name) - {} - - ~movable_exception() throw() {} - - const movable_exception& operator= ( const movable_exception& src ) { - if ( this != &src ) { - my_exception_data = src.my_exception_data; - my_exception_name = src.my_exception_name; - } - return *this; - } - - ExceptionData& data() throw() { return my_exception_data; } - - const ExceptionData& data() const throw() { return my_exception_data; } - - const char* name() const throw() __TBB_override { return my_exception_name; } - - const char* what() const throw() __TBB_override { return "tbb::movable_exception"; } - - movable_exception* move() throw() __TBB_override { - void* e = internal::allocate_via_handler_v3(sizeof(movable_exception)); - if ( e ) { - ::new (e) movable_exception(*this); - ((movable_exception*)e)->my_dynamic = true; - } - return (movable_exception*)e; - } - void destroy() throw() __TBB_override { - __TBB_ASSERT ( my_dynamic, "Method destroy can be called only on dynamically allocated movable_exceptions" ); - if ( my_dynamic ) { - this->~movable_exception(); - internal::deallocate_via_handler_v3(this); - } - } - void throw_self() __TBB_override { __TBB_THROW( *this ); } - -protected: - //! User data - ExceptionData my_exception_data; - -private: - //! Flag specifying whether this object has been dynamically allocated (by the move method) - bool my_dynamic; - - //! RTTI name of this class - /** We rely on the fact that RTTI names are static string constants. **/ - const char* my_exception_name; -}; - -#if !TBB_USE_CAPTURED_EXCEPTION -namespace internal { - -//! Exception container that preserves the exact copy of the original exception -/** This class can be used only when the appropriate runtime support (mandated - by C++11) is present **/ -class tbb_exception_ptr { - std::exception_ptr my_ptr; - -public: - static tbb_exception_ptr* allocate(); - static tbb_exception_ptr* allocate( const tbb_exception& tag ); - //! This overload uses move semantics (i.e. it empties src) - static tbb_exception_ptr* allocate( captured_exception& src ); - - //! Destroys this objects - /** Note that objects of this type can be created only by the allocate() method. **/ - void destroy() throw(); - - //! Throws the contained exception . - void throw_self() { std::rethrow_exception(my_ptr); } - -private: - tbb_exception_ptr( const std::exception_ptr& src ) : my_ptr(src) {} - tbb_exception_ptr( const captured_exception& src ) : - #if __TBB_MAKE_EXCEPTION_PTR_PRESENT - my_ptr(std::make_exception_ptr(src)) // the final function name in C++11 - #else - my_ptr(std::copy_exception(src)) // early C++0x drafts name - #endif - {} -}; // class tbb::internal::tbb_exception_ptr - -} // namespace internal -#endif /* !TBB_USE_CAPTURED_EXCEPTION */ - -} // namespace tbb - -#endif /* __TBB_TASK_GROUP_CONTEXT */ - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_tbb_exception_H_include_area - -#endif /* __TBB_exception_H */ diff --git a/src/3rd party/tbb/tbb_machine.h b/src/3rd party/tbb/tbb_machine.h deleted file mode 100644 index 095b9d3caa..0000000000 --- a/src/3rd party/tbb/tbb_machine.h +++ /dev/null @@ -1,978 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_machine_H -#define __TBB_machine_H - -/** This header provides basic platform abstraction layer by hooking up appropriate - architecture/OS/compiler specific headers from the /include/tbb/machine directory. - If a plug-in header does not implement all the required APIs, it must specify - the missing ones by setting one or more of the following macros: - - __TBB_USE_GENERIC_PART_WORD_CAS - __TBB_USE_GENERIC_PART_WORD_FETCH_ADD - __TBB_USE_GENERIC_PART_WORD_FETCH_STORE - __TBB_USE_GENERIC_FETCH_ADD - __TBB_USE_GENERIC_FETCH_STORE - __TBB_USE_GENERIC_DWORD_FETCH_ADD - __TBB_USE_GENERIC_DWORD_FETCH_STORE - __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE - __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE - __TBB_USE_GENERIC_RELAXED_LOAD_STORE - __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE - - In this case tbb_machine.h will add missing functionality based on a minimal set - of APIs that are required to be implemented by all plug-n headers as described - further. - Note that these generic implementations may be sub-optimal for a particular - architecture, and thus should be relied upon only after careful evaluation - or as the last resort. - - Additionally __TBB_64BIT_ATOMICS can be set to 0 on a 32-bit architecture to - indicate that the port is not going to support double word atomics. It may also - be set to 1 explicitly, though normally this is not necessary as tbb_machine.h - will set it automatically. - - __TBB_ENDIANNESS macro can be defined by the implementation as well. - It is used only if __TBB_USE_GENERIC_PART_WORD_CAS is set (or for testing), - and must specify the layout of aligned 16-bit and 32-bit data anywhere within a process - (while the details of unaligned 16-bit or 32-bit data or of 64-bit data are irrelevant). - The layout must be the same at all relevant memory locations within the current process; - in case of page-specific endianness, one endianness must be kept "out of sight". - Possible settings, reflecting hardware and possibly O.S. convention, are: - - __TBB_ENDIAN_BIG for big-endian data, - - __TBB_ENDIAN_LITTLE for little-endian data, - - __TBB_ENDIAN_DETECT for run-time detection iff exactly one of the above, - - __TBB_ENDIAN_UNSUPPORTED to prevent undefined behavior if none of the above. - - Prerequisites for each architecture port - ---------------------------------------- - The following functions and macros have no generic implementation. Therefore they must be - implemented in each machine architecture specific header either as a conventional - function or as a functional macro. - - __TBB_WORDSIZE - This is the size of machine word in bytes, i.e. for 32 bit systems it - should be defined to 4. - - __TBB_Yield() - Signals OS that the current thread is willing to relinquish the remainder - of its time quantum. - - __TBB_full_memory_fence() - Must prevent all memory operations from being reordered across it (both - by hardware and compiler). All such fences must be totally ordered (or - sequentially consistent). - - __TBB_machine_cmpswp4( volatile void *ptr, int32_t value, int32_t comparand ) - Must be provided if __TBB_USE_FENCED_ATOMICS is not set. - - __TBB_machine_cmpswp8( volatile void *ptr, int32_t value, int64_t comparand ) - Must be provided for 64-bit architectures if __TBB_USE_FENCED_ATOMICS is not set, - and for 32-bit architectures if __TBB_64BIT_ATOMICS is set - - __TBB_machine_(...), where - = {cmpswp, fetchadd, fetchstore} - = {1, 2, 4, 8} - = {full_fence, acquire, release, relaxed} - Must be provided if __TBB_USE_FENCED_ATOMICS is set. - - __TBB_control_consistency_helper() - Bridges the memory-semantics gap between architectures providing only - implicit C++0x "consume" semantics (like Power Architecture) and those - also implicitly obeying control dependencies (like IA-64 architecture). - It must be used only in conditional code where the condition is itself - data-dependent, and will then make subsequent code behave as if the - original data dependency were acquired. - It needs only a compiler fence where implied by the architecture - either specifically (like IA-64 architecture) or because generally stronger - "acquire" semantics are enforced (like x86). - It is always valid, though potentially suboptimal, to replace - control with acquire on the load and then remove the helper. - - __TBB_acquire_consistency_helper(), __TBB_release_consistency_helper() - Must be provided if __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE is set. - Enforce acquire and release semantics in generic implementations of fenced - store and load operations. Depending on the particular architecture/compiler - combination they may be a hardware fence, a compiler fence, both or nothing. - **/ - -#include "tbb_stddef.h" - -namespace tbb { -namespace internal { //< @cond INTERNAL - -//////////////////////////////////////////////////////////////////////////////// -// Overridable helpers declarations -// -// A machine/*.h file may choose to define these templates, otherwise it must -// request default implementation by setting appropriate __TBB_USE_GENERIC_XXX macro(s). -// -template -struct machine_load_store; - -template -struct machine_load_store_relaxed; - -template -struct machine_load_store_seq_cst; -// -// End of overridable helpers declarations -//////////////////////////////////////////////////////////////////////////////// - -template struct atomic_selector; - -template<> struct atomic_selector<1> { - typedef int8_t word; - inline static word fetch_store ( volatile void* location, word value ); -}; - -template<> struct atomic_selector<2> { - typedef int16_t word; - inline static word fetch_store ( volatile void* location, word value ); -}; - -template<> struct atomic_selector<4> { -#if _MSC_VER && !_WIN64 - // Work-around that avoids spurious /Wp64 warnings - typedef intptr_t word; -#else - typedef int32_t word; -#endif - inline static word fetch_store ( volatile void* location, word value ); -}; - -template<> struct atomic_selector<8> { - typedef int64_t word; - inline static word fetch_store ( volatile void* location, word value ); -}; - -}} //< namespaces internal @endcond, tbb - -#define __TBB_MACHINE_DEFINE_STORE8_GENERIC_FENCED(M) \ - inline void __TBB_machine_generic_store8##M(volatile void *ptr, int64_t value) { \ - for(;;) { \ - int64_t result = *(volatile int64_t *)ptr; \ - if( __TBB_machine_cmpswp8##M(ptr,value,result)==result ) break; \ - } \ - } \ - -#define __TBB_MACHINE_DEFINE_LOAD8_GENERIC_FENCED(M) \ - inline int64_t __TBB_machine_generic_load8##M(const volatile void *ptr) { \ - /* Comparand and new value may be anything, they only must be equal, and */ \ - /* the value should have a low probability to be actually found in 'location'.*/ \ - const int64_t anyvalue = 2305843009213693951LL; \ - return __TBB_machine_cmpswp8##M(const_cast(ptr),anyvalue,anyvalue); \ - } \ - -// The set of allowed values for __TBB_ENDIANNESS (see above for details) -#define __TBB_ENDIAN_UNSUPPORTED -1 -#define __TBB_ENDIAN_LITTLE 0 -#define __TBB_ENDIAN_BIG 1 -#define __TBB_ENDIAN_DETECT 2 - -#if _WIN32||_WIN64 - -#ifdef _MANAGED -#pragma managed(push, off) -#endif - - #if __MINGW64__ || __MINGW32__ - extern "C" __declspec(dllimport) int __stdcall SwitchToThread( void ); - #define __TBB_Yield() SwitchToThread() - #if (TBB_USE_GCC_BUILTINS && __TBB_GCC_BUILTIN_ATOMICS_PRESENT) - #include "machine/gcc_generic.h" - #elif __MINGW64__ - #include "machine/linux_intel64.h" - #elif __MINGW32__ - #include "machine/linux_ia32.h" - #endif - #elif (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT) - #include "machine/icc_generic.h" - #elif defined(_M_IX86) && !defined(__TBB_WIN32_USE_CL_BUILTINS) - #include "machine/windows_ia32.h" - #elif defined(_M_X64) - #include "machine/windows_intel64.h" - #elif defined(_M_ARM) || defined(__TBB_WIN32_USE_CL_BUILTINS) - #include "machine/msvc_armv7.h" - #endif - -#ifdef _MANAGED -#pragma managed(pop) -#endif - -#elif __TBB_DEFINE_MIC - - #include "machine/mic_common.h" - #if (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT) - #include "machine/icc_generic.h" - #else - #include "machine/linux_intel64.h" - #endif - -#elif __linux__ || __FreeBSD__ || __NetBSD__ || __OpenBSD__ - - #if (TBB_USE_GCC_BUILTINS && __TBB_GCC_BUILTIN_ATOMICS_PRESENT) - #include "machine/gcc_generic.h" - #elif (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT) - #include "machine/icc_generic.h" - #elif __i386__ - #include "machine/linux_ia32.h" - #elif __x86_64__ - #include "machine/linux_intel64.h" - #elif __ia64__ - #include "machine/linux_ia64.h" - #elif __powerpc__ - #include "machine/mac_ppc.h" - #elif __ARM_ARCH_7A__ || __aarch64__ - #include "machine/gcc_arm.h" - #elif __TBB_GCC_BUILTIN_ATOMICS_PRESENT - #include "machine/gcc_generic.h" - #endif - #include "machine/linux_common.h" - -#elif __APPLE__ - //TODO: TBB_USE_GCC_BUILTINS is not used for Mac, Sun, Aix - #if (TBB_USE_ICC_BUILTINS && __TBB_ICC_BUILTIN_ATOMICS_PRESENT) - #include "machine/icc_generic.h" - #elif __TBB_x86_32 - #include "machine/linux_ia32.h" - #elif __TBB_x86_64 - #include "machine/linux_intel64.h" - #elif __POWERPC__ - #include "machine/mac_ppc.h" - #endif - #include "machine/macos_common.h" - -#elif _AIX - - #include "machine/ibm_aix51.h" - -#elif __sun || __SUNPRO_CC - - #define __asm__ asm - #define __volatile__ volatile - - #if __i386 || __i386__ - #include "machine/linux_ia32.h" - #elif __x86_64__ - #include "machine/linux_intel64.h" - #elif __sparc - #include "machine/sunos_sparc.h" - #endif - #include - - #define __TBB_Yield() sched_yield() - -#endif /* OS selection */ - -#ifndef __TBB_64BIT_ATOMICS - #define __TBB_64BIT_ATOMICS 1 -#endif - -//TODO: replace usage of these functions with usage of tbb::atomic, and then remove them -//TODO: map functions with W suffix to use cast to tbb::atomic and according op, i.e. as_atomic().op() -// Special atomic functions -#if __TBB_USE_FENCED_ATOMICS - #define __TBB_machine_cmpswp1 __TBB_machine_cmpswp1full_fence - #define __TBB_machine_cmpswp2 __TBB_machine_cmpswp2full_fence - #define __TBB_machine_cmpswp4 __TBB_machine_cmpswp4full_fence - #define __TBB_machine_cmpswp8 __TBB_machine_cmpswp8full_fence - - #if __TBB_WORDSIZE==8 - #define __TBB_machine_fetchadd8 __TBB_machine_fetchadd8full_fence - #define __TBB_machine_fetchstore8 __TBB_machine_fetchstore8full_fence - #define __TBB_FetchAndAddWrelease(P,V) __TBB_machine_fetchadd8release(P,V) - #define __TBB_FetchAndIncrementWacquire(P) __TBB_machine_fetchadd8acquire(P,1) - #define __TBB_FetchAndDecrementWrelease(P) __TBB_machine_fetchadd8release(P,(-1)) - #else - #define __TBB_machine_fetchadd4 __TBB_machine_fetchadd4full_fence - #define __TBB_machine_fetchstore4 __TBB_machine_fetchstore4full_fence - #define __TBB_FetchAndAddWrelease(P,V) __TBB_machine_fetchadd4release(P,V) - #define __TBB_FetchAndIncrementWacquire(P) __TBB_machine_fetchadd4acquire(P,1) - #define __TBB_FetchAndDecrementWrelease(P) __TBB_machine_fetchadd4release(P,(-1)) - #endif /* __TBB_WORDSIZE==4 */ -#else /* !__TBB_USE_FENCED_ATOMICS */ - #define __TBB_FetchAndAddWrelease(P,V) __TBB_FetchAndAddW(P,V) - #define __TBB_FetchAndIncrementWacquire(P) __TBB_FetchAndAddW(P,1) - #define __TBB_FetchAndDecrementWrelease(P) __TBB_FetchAndAddW(P,(-1)) -#endif /* !__TBB_USE_FENCED_ATOMICS */ - -#if __TBB_WORDSIZE==4 - #define __TBB_CompareAndSwapW(P,V,C) __TBB_machine_cmpswp4(P,V,C) - #define __TBB_FetchAndAddW(P,V) __TBB_machine_fetchadd4(P,V) - #define __TBB_FetchAndStoreW(P,V) __TBB_machine_fetchstore4(P,V) -#elif __TBB_WORDSIZE==8 - #if __TBB_USE_GENERIC_DWORD_LOAD_STORE || __TBB_USE_GENERIC_DWORD_FETCH_ADD || __TBB_USE_GENERIC_DWORD_FETCH_STORE - #error These macros should only be used on 32-bit platforms. - #endif - - #define __TBB_CompareAndSwapW(P,V,C) __TBB_machine_cmpswp8(P,V,C) - #define __TBB_FetchAndAddW(P,V) __TBB_machine_fetchadd8(P,V) - #define __TBB_FetchAndStoreW(P,V) __TBB_machine_fetchstore8(P,V) -#else /* __TBB_WORDSIZE != 8 */ - #error Unsupported machine word size. -#endif /* __TBB_WORDSIZE */ - -#ifndef __TBB_Pause - inline void __TBB_Pause(int32_t) { - __TBB_Yield(); - } -#endif - -namespace tbb { - -//! Sequentially consistent full memory fence. -inline void atomic_fence () { __TBB_full_memory_fence(); } - -namespace internal { //< @cond INTERNAL - -//! Class that implements exponential backoff. -/** See implementation of spin_wait_while_eq for an example. */ -class atomic_backoff : no_copy { - //! Time delay, in units of "pause" instructions. - /** Should be equal to approximately the number of "pause" instructions - that take the same time as an context switch. Must be a power of two.*/ - static const int32_t LOOPS_BEFORE_YIELD = 16; - int32_t count; -public: - // In many cases, an object of this type is initialized eagerly on hot path, - // as in for(atomic_backoff b; ; b.pause()) { /*loop body*/ } - // For this reason, the construction cost must be very small! - atomic_backoff() : count(1) {} - // This constructor pauses immediately; do not use on hot paths! - atomic_backoff( bool ) : count(1) { pause(); } - - //! Pause for a while. - void pause() { - if( count<=LOOPS_BEFORE_YIELD ) { - __TBB_Pause(count); - // Pause twice as long the next time. - count*=2; - } else { - // Pause is so long that we might as well yield CPU to scheduler. - __TBB_Yield(); - } - } - - //! Pause for a few times and return false if saturated. - bool bounded_pause() { - __TBB_Pause(count); - if( count -void spin_wait_while_eq( const volatile T& location, U value ) { - atomic_backoff backoff; - while( location==value ) backoff.pause(); -} - -//! Spin UNTIL the value of the variable is equal to a given value -/** T and U should be comparable types. */ -template -void spin_wait_until_eq( const volatile T& location, const U value ) { - atomic_backoff backoff; - while( location!=value ) backoff.pause(); -} - -template -void spin_wait_while(predicate_type condition){ - atomic_backoff backoff; - while( condition() ) backoff.pause(); -} - -//////////////////////////////////////////////////////////////////////////////// -// Generic compare-and-swap applied to only a part of a machine word. -// -#ifndef __TBB_ENDIANNESS -#define __TBB_ENDIANNESS __TBB_ENDIAN_DETECT -#endif - -#if __TBB_USE_GENERIC_PART_WORD_CAS && __TBB_ENDIANNESS==__TBB_ENDIAN_UNSUPPORTED -#error Generic implementation of part-word CAS may not be used with __TBB_ENDIAN_UNSUPPORTED -#endif - -#if __TBB_ENDIANNESS!=__TBB_ENDIAN_UNSUPPORTED -// -// This function is the only use of __TBB_ENDIANNESS. -// The following restrictions/limitations apply for this operation: -// - T must be an integer type of at most 4 bytes for the casts and calculations to work -// - T must also be less than 4 bytes to avoid compiler warnings when computing mask -// (and for the operation to be useful at all, so no workaround is applied) -// - the architecture must consistently use either little-endian or big-endian (same for all locations) -// -// TODO: static_assert for the type requirements stated above -template -inline T __TBB_MaskedCompareAndSwap (volatile T * const ptr, const T value, const T comparand ) { - struct endianness{ static bool is_big_endian(){ - #if __TBB_ENDIANNESS==__TBB_ENDIAN_DETECT - const uint32_t probe = 0x03020100; - return (((const char*)(&probe))[0]==0x03); - #elif __TBB_ENDIANNESS==__TBB_ENDIAN_BIG || __TBB_ENDIANNESS==__TBB_ENDIAN_LITTLE - return __TBB_ENDIANNESS==__TBB_ENDIAN_BIG; - #else - #error Unexpected value of __TBB_ENDIANNESS - #endif - }}; - - const uint32_t byte_offset = (uint32_t) ((uintptr_t)ptr & 0x3); - volatile uint32_t * const aligned_ptr = (uint32_t*)((uintptr_t)ptr - byte_offset ); - - // location of T within uint32_t for a C++ shift operation - const uint32_t bits_to_shift = 8*(endianness::is_big_endian() ? (4 - sizeof(T) - (byte_offset)) : byte_offset); - const uint32_t mask = (((uint32_t)1<<(sizeof(T)*8)) - 1 )<> bits_to_shift); - } - else continue; // CAS failed but the bits of interest were not changed - } -} -#endif // __TBB_ENDIANNESS!=__TBB_ENDIAN_UNSUPPORTED -//////////////////////////////////////////////////////////////////////////////// - -template -inline T __TBB_CompareAndSwapGeneric (volatile void *ptr, T value, T comparand ); - -template<> -inline int8_t __TBB_CompareAndSwapGeneric <1,int8_t> (volatile void *ptr, int8_t value, int8_t comparand ) { -#if __TBB_USE_GENERIC_PART_WORD_CAS - return __TBB_MaskedCompareAndSwap((volatile int8_t *)ptr,value,comparand); -#else - return __TBB_machine_cmpswp1(ptr,value,comparand); -#endif -} - -template<> -inline int16_t __TBB_CompareAndSwapGeneric <2,int16_t> (volatile void *ptr, int16_t value, int16_t comparand ) { -#if __TBB_USE_GENERIC_PART_WORD_CAS - return __TBB_MaskedCompareAndSwap((volatile int16_t *)ptr,value,comparand); -#else - return __TBB_machine_cmpswp2(ptr,value,comparand); -#endif -} - -template<> -inline int32_t __TBB_CompareAndSwapGeneric <4,int32_t> (volatile void *ptr, int32_t value, int32_t comparand ) { - // Cast shuts up /Wp64 warning - return (int32_t)__TBB_machine_cmpswp4(ptr,value,comparand); -} - -#if __TBB_64BIT_ATOMICS -template<> -inline int64_t __TBB_CompareAndSwapGeneric <8,int64_t> (volatile void *ptr, int64_t value, int64_t comparand ) { - return __TBB_machine_cmpswp8(ptr,value,comparand); -} -#endif - -template -inline T __TBB_FetchAndAddGeneric (volatile void *ptr, T addend) { - T result; - for( atomic_backoff b;;b.pause() ) { - result = *reinterpret_cast(ptr); - // __TBB_CompareAndSwapGeneric presumed to have full fence. - if( __TBB_CompareAndSwapGeneric ( ptr, result+addend, result )==result ) - break; - } - return result; -} - -template -inline T __TBB_FetchAndStoreGeneric (volatile void *ptr, T value) { - T result; - for( atomic_backoff b;;b.pause() ) { - result = *reinterpret_cast(ptr); - // __TBB_CompareAndSwapGeneric presumed to have full fence. - if( __TBB_CompareAndSwapGeneric ( ptr, value, result )==result ) - break; - } - return result; -} - -#if __TBB_USE_GENERIC_PART_WORD_CAS -#define __TBB_machine_cmpswp1 tbb::internal::__TBB_CompareAndSwapGeneric<1,int8_t> -#define __TBB_machine_cmpswp2 tbb::internal::__TBB_CompareAndSwapGeneric<2,int16_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_ADD || __TBB_USE_GENERIC_PART_WORD_FETCH_ADD -#define __TBB_machine_fetchadd1 tbb::internal::__TBB_FetchAndAddGeneric<1,int8_t> -#define __TBB_machine_fetchadd2 tbb::internal::__TBB_FetchAndAddGeneric<2,int16_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_ADD -#define __TBB_machine_fetchadd4 tbb::internal::__TBB_FetchAndAddGeneric<4,int32_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_ADD || __TBB_USE_GENERIC_DWORD_FETCH_ADD -#define __TBB_machine_fetchadd8 tbb::internal::__TBB_FetchAndAddGeneric<8,int64_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_STORE || __TBB_USE_GENERIC_PART_WORD_FETCH_STORE -#define __TBB_machine_fetchstore1 tbb::internal::__TBB_FetchAndStoreGeneric<1,int8_t> -#define __TBB_machine_fetchstore2 tbb::internal::__TBB_FetchAndStoreGeneric<2,int16_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_STORE -#define __TBB_machine_fetchstore4 tbb::internal::__TBB_FetchAndStoreGeneric<4,int32_t> -#endif - -#if __TBB_USE_GENERIC_FETCH_STORE || __TBB_USE_GENERIC_DWORD_FETCH_STORE -#define __TBB_machine_fetchstore8 tbb::internal::__TBB_FetchAndStoreGeneric<8,int64_t> -#endif - -#if __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE -#define __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(S) \ - atomic_selector::word atomic_selector::fetch_store ( volatile void* location, word value ) { \ - return __TBB_machine_fetchstore##S( location, value ); \ - } - -__TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(1) -__TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(2) -__TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(4) -__TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE(8) - -#undef __TBB_MACHINE_DEFINE_ATOMIC_SELECTOR_FETCH_STORE -#endif /* __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE */ - -#if __TBB_USE_GENERIC_DWORD_LOAD_STORE -/*TODO: find a more elegant way to handle function names difference*/ -#if ! __TBB_USE_FENCED_ATOMICS - /* This name forwarding is needed for generic implementation of - * load8/store8 defined below (via macro) to pick the right CAS function*/ - #define __TBB_machine_cmpswp8full_fence __TBB_machine_cmpswp8 -#endif -__TBB_MACHINE_DEFINE_LOAD8_GENERIC_FENCED(full_fence) -__TBB_MACHINE_DEFINE_STORE8_GENERIC_FENCED(full_fence) - -#if ! __TBB_USE_FENCED_ATOMICS - #undef __TBB_machine_cmpswp8full_fence -#endif - -#define __TBB_machine_store8 tbb::internal::__TBB_machine_generic_store8full_fence -#define __TBB_machine_load8 tbb::internal::__TBB_machine_generic_load8full_fence -#endif /* __TBB_USE_GENERIC_DWORD_LOAD_STORE */ - -#if __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE -/** Fenced operations use volatile qualifier to prevent compiler from optimizing - them out, and on architectures with weak memory ordering to induce compiler - to generate code with appropriate acquire/release semantics. - On architectures like IA32, Intel64 (and likely Sparc TSO) volatile has - no effect on code gen, and consistency helpers serve as a compiler fence (the - latter being true for IA64/gcc as well to fix a bug in some gcc versions). - This code assumes that the generated instructions will operate atomically, - which typically requires a type that can be moved in a single instruction, - cooperation from the compiler for effective use of such an instruction, - and appropriate alignment of the data. **/ -template -struct machine_load_store { - static T load_with_acquire ( const volatile T& location ) { - T to_return = location; - __TBB_acquire_consistency_helper(); - return to_return; - } - static void store_with_release ( volatile T &location, T value ) { - __TBB_release_consistency_helper(); - location = value; - } -}; - -//in general, plain load and store of 32bit compiler is not atomic for 64bit types -#if __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS -template -struct machine_load_store { - static T load_with_acquire ( const volatile T& location ) { - return (T)__TBB_machine_load8( (const volatile void*)&location ); - } - static void store_with_release ( volatile T& location, T value ) { - __TBB_machine_store8( (volatile void*)&location, (int64_t)value ); - } -}; -#endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */ -#endif /* __TBB_USE_GENERIC_HALF_FENCED_LOAD_STORE */ - -#if __TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE -template -struct machine_load_store_seq_cst { - static T load ( const volatile T& location ) { - __TBB_full_memory_fence(); - return machine_load_store::load_with_acquire( location ); - } -#if __TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE - static void store ( volatile T &location, T value ) { - atomic_selector::fetch_store( (volatile void*)&location, (typename atomic_selector::word)value ); - } -#else /* !__TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE */ - static void store ( volatile T &location, T value ) { - machine_load_store::store_with_release( location, value ); - __TBB_full_memory_fence(); - } -#endif /* !__TBB_USE_FETCHSTORE_AS_FULL_FENCED_STORE */ -}; - -#if __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS -/** The implementation does not use functions __TBB_machine_load8/store8 as they - are not required to be sequentially consistent. **/ -template -struct machine_load_store_seq_cst { - static T load ( const volatile T& location ) { - // Comparand and new value may be anything, they only must be equal, and - // the value should have a low probability to be actually found in 'location'. - const int64_t anyvalue = 2305843009213693951LL; - return __TBB_machine_cmpswp8( (volatile void*)const_cast(&location), anyvalue, anyvalue ); - } - static void store ( volatile T &location, T value ) { -#if __TBB_GCC_VERSION >= 40702 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - // An atomic initialization leads to reading of uninitialized memory - int64_t result = (volatile int64_t&)location; -#if __TBB_GCC_VERSION >= 40702 -#pragma GCC diagnostic pop -#endif - while ( __TBB_machine_cmpswp8((volatile void*)&location, (int64_t)value, result) != result ) - result = (volatile int64_t&)location; - } -}; -#endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */ -#endif /*__TBB_USE_GENERIC_SEQUENTIAL_CONSISTENCY_LOAD_STORE */ - -#if __TBB_USE_GENERIC_RELAXED_LOAD_STORE -// Relaxed operations add volatile qualifier to prevent compiler from optimizing them out. -/** Volatile should not incur any additional cost on IA32, Intel64, and Sparc TSO - architectures. However on architectures with weak memory ordering compiler may - generate code with acquire/release semantics for operations on volatile data. **/ -template -struct machine_load_store_relaxed { - static inline T load ( const volatile T& location ) { - return location; - } - static inline void store ( volatile T& location, T value ) { - location = value; - } -}; - -#if __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS -template -struct machine_load_store_relaxed { - static inline T load ( const volatile T& location ) { - return (T)__TBB_machine_load8( (const volatile void*)&location ); - } - static inline void store ( volatile T& location, T value ) { - __TBB_machine_store8( (volatile void*)&location, (int64_t)value ); - } -}; -#endif /* __TBB_WORDSIZE==4 && __TBB_64BIT_ATOMICS */ -#endif /* __TBB_USE_GENERIC_RELAXED_LOAD_STORE */ - -#undef __TBB_WORDSIZE //this macro is forbidden to use outside of atomic machinery - -template -inline T __TBB_load_with_acquire(const volatile T &location) { - return machine_load_store::load_with_acquire( location ); -} -template -inline void __TBB_store_with_release(volatile T& location, V value) { - machine_load_store::store_with_release( location, T(value) ); -} -//! Overload that exists solely to avoid /Wp64 warnings. -inline void __TBB_store_with_release(volatile size_t& location, size_t value) { - machine_load_store::store_with_release( location, value ); -} - -template -inline T __TBB_load_full_fence(const volatile T &location) { - return machine_load_store_seq_cst::load( location ); -} -template -inline void __TBB_store_full_fence(volatile T& location, V value) { - machine_load_store_seq_cst::store( location, T(value) ); -} -//! Overload that exists solely to avoid /Wp64 warnings. -inline void __TBB_store_full_fence(volatile size_t& location, size_t value) { - machine_load_store_seq_cst::store( location, value ); -} - -template -inline T __TBB_load_relaxed (const volatile T& location) { - return machine_load_store_relaxed::load( const_cast(location) ); -} -template -inline void __TBB_store_relaxed ( volatile T& location, V value ) { - machine_load_store_relaxed::store( const_cast(location), T(value) ); -} -//! Overload that exists solely to avoid /Wp64 warnings. -inline void __TBB_store_relaxed ( volatile size_t& location, size_t value ) { - machine_load_store_relaxed::store( const_cast(location), value ); -} - -// Macro __TBB_TypeWithAlignmentAtLeastAsStrict(T) should be a type with alignment at least as -// strict as type T. The type should have a trivial default constructor and destructor, so that -// arrays of that type can be declared without initializers. -// It is correct (but perhaps a waste of space) if __TBB_TypeWithAlignmentAtLeastAsStrict(T) expands -// to a type bigger than T. -// The default definition here works on machines where integers are naturally aligned and the -// strictest alignment is 64. -#ifndef __TBB_TypeWithAlignmentAtLeastAsStrict - -#if __TBB_ALIGNAS_PRESENT - -// Use C++11 keywords alignas and alignof -#define __TBB_DefineTypeWithAlignment(PowerOf2) \ -struct alignas(PowerOf2) __TBB_machine_type_with_alignment_##PowerOf2 { \ - uint32_t member[PowerOf2/sizeof(uint32_t)]; \ -}; -#define __TBB_alignof(T) alignof(T) - -#elif __TBB_ATTRIBUTE_ALIGNED_PRESENT - -#define __TBB_DefineTypeWithAlignment(PowerOf2) \ -struct __TBB_machine_type_with_alignment_##PowerOf2 { \ - uint32_t member[PowerOf2/sizeof(uint32_t)]; \ -} __attribute__((aligned(PowerOf2))); -#define __TBB_alignof(T) __alignof__(T) - -#elif __TBB_DECLSPEC_ALIGN_PRESENT - -#define __TBB_DefineTypeWithAlignment(PowerOf2) \ -__declspec(align(PowerOf2)) \ -struct __TBB_machine_type_with_alignment_##PowerOf2 { \ - uint32_t member[PowerOf2/sizeof(uint32_t)]; \ -}; -#define __TBB_alignof(T) __alignof(T) - -#else /* A compiler with unknown syntax for data alignment */ -#error Must define __TBB_TypeWithAlignmentAtLeastAsStrict(T) -#endif - -/* Now declare types aligned to useful powers of two */ -__TBB_DefineTypeWithAlignment(8) // i386 ABI says that uint64_t is aligned on 4 bytes -__TBB_DefineTypeWithAlignment(16) -__TBB_DefineTypeWithAlignment(32) -__TBB_DefineTypeWithAlignment(64) - -typedef __TBB_machine_type_with_alignment_64 __TBB_machine_type_with_strictest_alignment; - -// Primary template is a declaration of incomplete type so that it fails with unknown alignments -template struct type_with_alignment; - -// Specializations for allowed alignments -template<> struct type_with_alignment<1> { char member; }; -template<> struct type_with_alignment<2> { uint16_t member; }; -template<> struct type_with_alignment<4> { uint32_t member; }; -template<> struct type_with_alignment<8> { __TBB_machine_type_with_alignment_8 member; }; -template<> struct type_with_alignment<16> {__TBB_machine_type_with_alignment_16 member; }; -template<> struct type_with_alignment<32> {__TBB_machine_type_with_alignment_32 member; }; -template<> struct type_with_alignment<64> {__TBB_machine_type_with_alignment_64 member; }; - -#if __TBB_ALIGNOF_NOT_INSTANTIATED_TYPES_BROKEN -//! Work around for bug in GNU 3.2 and MSVC compilers. -/** Bug is that compiler sometimes returns 0 for __alignof(T) when T has not yet been instantiated. - The work-around forces instantiation by forcing computation of sizeof(T) before __alignof(T). */ -template -struct work_around_alignment_bug { - static const size_t alignment = __TBB_alignof(T); -}; -#define __TBB_TypeWithAlignmentAtLeastAsStrict(T) tbb::internal::type_with_alignment::alignment> -#else -#define __TBB_TypeWithAlignmentAtLeastAsStrict(T) tbb::internal::type_with_alignment<__TBB_alignof(T)> -#endif /* __TBB_ALIGNOF_NOT_INSTANTIATED_TYPES_BROKEN */ - -#endif /* __TBB_TypeWithAlignmentAtLeastAsStrict */ - -// Template class here is to avoid instantiation of the static data for modules that don't use it -template -struct reverse { - static const T byte_table[256]; -}; -// An efficient implementation of the reverse function utilizes a 2^8 lookup table holding the bit-reversed -// values of [0..2^8 - 1]. Those values can also be computed on the fly at a slightly higher cost. -template -const T reverse::byte_table[256] = { - 0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0, 0x10, 0x90, 0x50, 0xD0, 0x30, 0xB0, 0x70, 0xF0, - 0x08, 0x88, 0x48, 0xC8, 0x28, 0xA8, 0x68, 0xE8, 0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, 0x78, 0xF8, - 0x04, 0x84, 0x44, 0xC4, 0x24, 0xA4, 0x64, 0xE4, 0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4, - 0x0C, 0x8C, 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC, 0x1C, 0x9C, 0x5C, 0xDC, 0x3C, 0xBC, 0x7C, 0xFC, - 0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, 0x62, 0xE2, 0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2, - 0x0A, 0x8A, 0x4A, 0xCA, 0x2A, 0xAA, 0x6A, 0xEA, 0x1A, 0x9A, 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA, - 0x06, 0x86, 0x46, 0xC6, 0x26, 0xA6, 0x66, 0xE6, 0x16, 0x96, 0x56, 0xD6, 0x36, 0xB6, 0x76, 0xF6, - 0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE, 0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE, - 0x01, 0x81, 0x41, 0xC1, 0x21, 0xA1, 0x61, 0xE1, 0x11, 0x91, 0x51, 0xD1, 0x31, 0xB1, 0x71, 0xF1, - 0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, 0x69, 0xE9, 0x19, 0x99, 0x59, 0xD9, 0x39, 0xB9, 0x79, 0xF9, - 0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5, 0x15, 0x95, 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5, - 0x0D, 0x8D, 0x4D, 0xCD, 0x2D, 0xAD, 0x6D, 0xED, 0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, 0x7D, 0xFD, - 0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3, 0x13, 0x93, 0x53, 0xD3, 0x33, 0xB3, 0x73, 0xF3, - 0x0B, 0x8B, 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB, 0x1B, 0x9B, 0x5B, 0xDB, 0x3B, 0xBB, 0x7B, 0xFB, - 0x07, 0x87, 0x47, 0xC7, 0x27, 0xA7, 0x67, 0xE7, 0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7, - 0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF, 0x1F, 0x9F, 0x5F, 0xDF, 0x3F, 0xBF, 0x7F, 0xFF -}; - -} // namespace internal @endcond -} // namespace tbb - -// Preserving access to legacy APIs -using tbb::internal::__TBB_load_with_acquire; -using tbb::internal::__TBB_store_with_release; - -// Mapping historically used names to the ones expected by atomic_load_store_traits -#define __TBB_load_acquire __TBB_load_with_acquire -#define __TBB_store_release __TBB_store_with_release - -#ifndef __TBB_Log2 -inline intptr_t __TBB_Log2( uintptr_t x ) { - if( x==0 ) return -1; - intptr_t result = 0; - -#if !defined(_M_ARM) - uintptr_t tmp_; - if( sizeof(x)>4 && (tmp_ = ((uint64_t)x)>>32) ) { x=tmp_; result += 32; } -#endif - if( uintptr_t tmp = x>>16 ) { x=tmp; result += 16; } - if( uintptr_t tmp = x>>8 ) { x=tmp; result += 8; } - if( uintptr_t tmp = x>>4 ) { x=tmp; result += 4; } - if( uintptr_t tmp = x>>2 ) { x=tmp; result += 2; } - - return (x&2)? result+1: result; -} -#endif - -#ifndef __TBB_AtomicOR -inline void __TBB_AtomicOR( volatile void *operand, uintptr_t addend ) { - for( tbb::internal::atomic_backoff b;;b.pause() ) { - uintptr_t tmp = *(volatile uintptr_t *)operand; - uintptr_t result = __TBB_CompareAndSwapW(operand, tmp|addend, tmp); - if( result==tmp ) break; - } -} -#endif - -#ifndef __TBB_AtomicAND -inline void __TBB_AtomicAND( volatile void *operand, uintptr_t addend ) { - for( tbb::internal::atomic_backoff b;;b.pause() ) { - uintptr_t tmp = *(volatile uintptr_t *)operand; - uintptr_t result = __TBB_CompareAndSwapW(operand, tmp&addend, tmp); - if( result==tmp ) break; - } -} -#endif - -#if __TBB_PREFETCHING -#ifndef __TBB_cl_prefetch -#error This platform does not define cache management primitives required for __TBB_PREFETCHING -#endif - -#ifndef __TBB_cl_evict -#define __TBB_cl_evict(p) -#endif -#endif - -#ifndef __TBB_Flag -typedef unsigned char __TBB_Flag; -#endif -typedef __TBB_atomic __TBB_Flag __TBB_atomic_flag; - -#ifndef __TBB_TryLockByte -inline bool __TBB_TryLockByte( __TBB_atomic_flag &flag ) { - return __TBB_machine_cmpswp1(&flag,1,0)==0; -} -#endif - -#ifndef __TBB_LockByte -inline __TBB_Flag __TBB_LockByte( __TBB_atomic_flag& flag ) { - tbb::internal::atomic_backoff backoff; - while( !__TBB_TryLockByte(flag) ) backoff.pause(); - return 0; -} -#endif - -#ifndef __TBB_UnlockByte -#define __TBB_UnlockByte(addr) __TBB_store_with_release((addr),0) -#endif - -// lock primitives with Intel(R) Transactional Synchronization Extensions (Intel(R) TSX) -#if ( __TBB_x86_32 || __TBB_x86_64 ) /* only on ia32/intel64 */ -inline void __TBB_TryLockByteElidedCancel() { __TBB_machine_try_lock_elided_cancel(); } - -inline bool __TBB_TryLockByteElided( __TBB_atomic_flag& flag ) { - bool res = __TBB_machine_try_lock_elided( &flag )!=0; - // to avoid the "lemming" effect, we need to abort the transaction - // if __TBB_machine_try_lock_elided returns false (i.e., someone else - // has acquired the mutex non-speculatively). - if( !res ) __TBB_TryLockByteElidedCancel(); - return res; -} - -inline void __TBB_LockByteElided( __TBB_atomic_flag& flag ) -{ - for(;;) { - tbb::internal::spin_wait_while_eq( flag, 1 ); - if( __TBB_machine_try_lock_elided( &flag ) ) - return; - // Another thread acquired the lock "for real". - // To avoid the "lemming" effect, we abort the transaction. - __TBB_TryLockByteElidedCancel(); - } -} - -inline void __TBB_UnlockByteElided( __TBB_atomic_flag& flag ) { - __TBB_machine_unlock_elided( &flag ); -} -#endif - -#ifndef __TBB_ReverseByte -inline unsigned char __TBB_ReverseByte(unsigned char src) { - return tbb::internal::reverse::byte_table[src]; -} -#endif - -template -T __TBB_ReverseBits(T src) { - T dst; - unsigned char *original = (unsigned char *) &src; - unsigned char *reversed = (unsigned char *) &dst; - - for( int i = sizeof(T)-1; i >= 0; i-- ) - reversed[i] = __TBB_ReverseByte( original[sizeof(T)-i-1] ); - - return dst; -} - -#endif /* __TBB_machine_H */ diff --git a/src/3rd party/tbb/tbb_profiling.h b/src/3rd party/tbb/tbb_profiling.h deleted file mode 100644 index 9536e617e0..0000000000 --- a/src/3rd party/tbb/tbb_profiling.h +++ /dev/null @@ -1,355 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_profiling_H -#define __TBB_profiling_H - -#define __TBB_tbb_profiling_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -namespace tbb { - namespace internal { - - // include list of index names - #define TBB_STRING_RESOURCE(index_name,str) index_name, - enum string_index { - #include "internal/_tbb_strings.h" - NUM_STRINGS - }; - #undef TBB_STRING_RESOURCE - - enum itt_relation - { - __itt_relation_is_unknown = 0, - __itt_relation_is_dependent_on, /**< "A is dependent on B" means that A cannot start until B completes */ - __itt_relation_is_sibling_of, /**< "A is sibling of B" means that A and B were created as a group */ - __itt_relation_is_parent_of, /**< "A is parent of B" means that A created B */ - __itt_relation_is_continuation_of, /**< "A is continuation of B" means that A assumes the dependencies of B */ - __itt_relation_is_child_of, /**< "A is child of B" means that A was created by B (inverse of is_parent_of) */ - __itt_relation_is_continued_by, /**< "A is continued by B" means that B assumes the dependencies of A (inverse of is_continuation_of) */ - __itt_relation_is_predecessor_to /**< "A is predecessor to B" means that B cannot start until A completes (inverse of is_dependent_on) */ - }; - - } -} - -// Check if the tools support is enabled -#if (_WIN32||_WIN64||__linux__) && !__MINGW32__ && TBB_USE_THREADING_TOOLS - -#if _WIN32||_WIN64 -#include /* mbstowcs_s */ -#endif -#include "tbb_stddef.h" - -namespace tbb { - namespace internal { - -#if _WIN32||_WIN64 - void __TBB_EXPORTED_FUNC itt_set_sync_name_v3( void *obj, const wchar_t* name ); - inline size_t multibyte_to_widechar( wchar_t* wcs, const char* mbs, size_t bufsize) { -#if _MSC_VER>=1400 - size_t len; - mbstowcs_s( &len, wcs, bufsize, mbs, _TRUNCATE ); - return len; // mbstowcs_s counts null terminator -#else - size_t len = mbstowcs( wcs, mbs, bufsize ); - if(wcs && len!=size_t(-1) ) - wcs[len - inline void itt_store_word_with_release(tbb::atomic& dst, U src) { -#if TBB_USE_THREADING_TOOLS - // This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized."); - itt_store_pointer_with_release_v3(&dst, (void *)uintptr_t(src)); -#else - dst = src; -#endif // TBB_USE_THREADING_TOOLS - } - - template - inline T itt_load_word_with_acquire(const tbb::atomic& src) { -#if TBB_USE_THREADING_TOOLS - // This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized."); -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - // Workaround for overzealous compiler warnings - #pragma warning (push) - #pragma warning (disable: 4311) -#endif - T result = (T)itt_load_pointer_with_acquire_v3(&src); -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) - #pragma warning (pop) -#endif - return result; -#else - return src; -#endif // TBB_USE_THREADING_TOOLS - } - - template - inline void itt_store_word_with_release(T& dst, T src) { -#if TBB_USE_THREADING_TOOLS - // This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized."); - itt_store_pointer_with_release_v3(&dst, (void *)src); -#else - __TBB_store_with_release(dst, src); -#endif // TBB_USE_THREADING_TOOLS - } - - template - inline T itt_load_word_with_acquire(const T& src) { -#if TBB_USE_THREADING_TOOLS - // This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized"); - return (T)itt_load_pointer_with_acquire_v3(&src); -#else - return __TBB_load_with_acquire(src); -#endif // TBB_USE_THREADING_TOOLS - } - - template - inline void itt_hide_store_word(T& dst, T src) { -#if TBB_USE_THREADING_TOOLS - //TODO: This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized"); - itt_store_pointer_with_release_v3(&dst, (void *)src); -#else - dst = src; -#endif - } - - //TODO: rename to itt_hide_load_word_relaxed - template - inline T itt_hide_load_word(const T& src) { -#if TBB_USE_THREADING_TOOLS - //TODO: This assertion should be replaced with static_assert - __TBB_ASSERT(sizeof(T) == sizeof(void *), "Type must be word-sized."); - return (T)itt_load_pointer_v3(&src); -#else - return src; -#endif - } - -#if TBB_USE_THREADING_TOOLS - inline void call_itt_notify(notify_type t, void *ptr) { - call_itt_notify_v5((int)t, ptr); - } - - inline void itt_make_task_group( itt_domain_enum domain, void *group, unsigned long long group_extra, - void *parent, unsigned long long parent_extra, string_index name_index ) { - itt_make_task_group_v7( domain, group, group_extra, parent, parent_extra, name_index ); - } - - inline void itt_metadata_str_add( itt_domain_enum domain, void *addr, unsigned long long addr_extra, - string_index key, const char *value ) { - itt_metadata_str_add_v7( domain, addr, addr_extra, key, value ); - } - - inline void register_node_addr(itt_domain_enum domain, void *addr, unsigned long long addr_extra, - string_index key, void *value) { - itt_metadata_ptr_add_v11(domain, addr, addr_extra, key, value); - } - - inline void itt_relation_add( itt_domain_enum domain, void *addr0, unsigned long long addr0_extra, - itt_relation relation, void *addr1, unsigned long long addr1_extra ) { - itt_relation_add_v7( domain, addr0, addr0_extra, relation, addr1, addr1_extra ); - } - - inline void itt_task_begin( itt_domain_enum domain, void *task, unsigned long long task_extra, - void *parent, unsigned long long parent_extra, string_index name_index ) { - itt_task_begin_v7( domain, task, task_extra, parent, parent_extra, name_index ); - } - - inline void itt_task_end( itt_domain_enum domain ) { - itt_task_end_v7( domain ); - } - - inline void itt_region_begin( itt_domain_enum domain, void *region, unsigned long long region_extra, - void *parent, unsigned long long parent_extra, string_index name_index ) { - itt_region_begin_v9( domain, region, region_extra, parent, parent_extra, name_index ); - } - - inline void itt_region_end( itt_domain_enum domain, void *region, unsigned long long region_extra ) { - itt_region_end_v9( domain, region, region_extra ); - } -#else - inline void register_node_addr( itt_domain_enum /*domain*/, void* /*addr*/, unsigned long long /*addr_extra*/, string_index /*key*/, void* /*value*/ ) {} - inline void call_itt_notify(notify_type /*t*/, void* /*ptr*/) {} - - inline void itt_make_task_group( itt_domain_enum /*domain*/, void* /*group*/, unsigned long long /*group_extra*/, - void* /*parent*/, unsigned long long /*parent_extra*/, string_index /*name_index*/ ) {} - - inline void itt_metadata_str_add( itt_domain_enum /*domain*/, void* /*addr*/, unsigned long long /*addr_extra*/, - string_index /*key*/, const char* /*value*/ ) {} - - inline void itt_relation_add( itt_domain_enum /*domain*/, void* /*addr0*/, unsigned long long /*addr0_extra*/, - itt_relation /*relation*/, void* /*addr1*/, unsigned long long /*addr1_extra*/ ) {} - - inline void itt_task_begin( itt_domain_enum /*domain*/, void* /*task*/, unsigned long long /*task_extra*/, - void* /*parent*/, unsigned long long /*parent_extra*/, string_index /*name_index*/ ) {} - - inline void itt_task_end( itt_domain_enum /*domain*/ ) {} - - inline void itt_region_begin( itt_domain_enum /*domain*/, void* /*region*/, unsigned long long /*region_extra*/, - void* /*parent*/, unsigned long long /*parent_extra*/, string_index /*name_index*/ ) {} - - inline void itt_region_end( itt_domain_enum /*domain*/, void* /*region*/, unsigned long long /*region_extra*/ ) {} -#endif // TBB_USE_THREADING_TOOLS - - } // namespace internal -} // namespace tbb - -#if TBB_PREVIEW_FLOW_GRAPH_TRACE -#include - -namespace tbb { -namespace profiling { -namespace interface10 { - -#if TBB_USE_THREADING_TOOLS && !(TBB_USE_THREADING_TOOLS == 2) -class event { -/** This class supports user event traces through itt. - Common use-case is tagging data flow graph tasks (data-id) - and visualization by Intel Advisor Flow Graph Analyzer (FGA) **/ -// TODO: Replace implementation by itt user event api. - - const std::string my_name; - - static void emit_trace(const std::string &input) { - itt_metadata_str_add( tbb::internal::ITT_DOMAIN_FLOW, NULL, tbb::internal::FLOW_NULL, tbb::internal::USER_EVENT, ( "FGA::DATAID::" + input ).c_str() ); - } - -public: - event(const std::string &input) - : my_name( input ) - { } - - void emit() { - emit_trace(my_name); - } - - static void emit(const std::string &description) { - emit_trace(description); - } - -}; -#else // TBB_USE_THREADING_TOOLS && !(TBB_USE_THREADING_TOOLS == 2) -// Using empty struct if user event tracing is disabled: -struct event { - event(const std::string &) { } - - void emit() { } - - static void emit(const std::string &) { } -}; -#endif // TBB_USE_THREADING_TOOLS && !(TBB_USE_THREADING_TOOLS == 2) - -} // interfaceX -using interface10::event; -} // namespace profiling -} // namespace tbb -#endif // TBB_PREVIEW_FLOW_GRAPH_TRACE - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_tbb_profiling_H_include_area - -#endif /* __TBB_profiling_H */ diff --git a/src/3rd party/tbb/tbb_stddef.h b/src/3rd party/tbb/tbb_stddef.h deleted file mode 100644 index 749ea1f45c..0000000000 --- a/src/3rd party/tbb/tbb_stddef.h +++ /dev/null @@ -1,565 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#ifndef __TBB_tbb_stddef_H -#define __TBB_tbb_stddef_H - -// Marketing-driven product version -#define TBB_VERSION_MAJOR 2020 -#define TBB_VERSION_MINOR 0 - -// Engineering-focused interface version -#define TBB_INTERFACE_VERSION 11100 -#define TBB_INTERFACE_VERSION_MAJOR TBB_INTERFACE_VERSION/1000 - -// The oldest major interface version still supported -// To be used in SONAME, manifests, etc. -#define TBB_COMPATIBLE_INTERFACE_VERSION 2 - -#define __TBB_STRING_AUX(x) #x -#define __TBB_STRING(x) __TBB_STRING_AUX(x) - -// We do not need defines below for resource processing on windows -#if !defined RC_INVOKED - -// Define groups for Doxygen documentation -/** - * @defgroup algorithms Algorithms - * @defgroup containers Containers - * @defgroup memory_allocation Memory Allocation - * @defgroup synchronization Synchronization - * @defgroup timing Timing - * @defgroup task_scheduling Task Scheduling - */ - -// Simple text that is displayed on the main page of Doxygen documentation. -/** - * \mainpage Main Page - * - * Click the tabs above for information about the - * - Modules (groups of functionality) implemented by the library - * - Classes provided by the library - * - Files constituting the library. - * . - * Please note that significant part of TBB functionality is implemented in the form of - * template functions, descriptions of which are not accessible on the Classes - * tab. Use Modules or Namespace/Namespace Members - * tabs to find them. - * - * Additional pieces of information can be found here - * - \subpage concepts - * . - */ - -/** \page concepts TBB concepts - - A concept is a set of requirements to a type, which are necessary and sufficient - for the type to model a particular behavior or a set of behaviors. Some concepts - are specific to a particular algorithm (e.g. algorithm body), while other ones - are common to several algorithms (e.g. range concept). - - All TBB algorithms make use of different classes implementing various concepts. - Implementation classes are supplied by the user as type arguments of template - parameters and/or as objects passed as function call arguments. The library - provides predefined implementations of some concepts (e.g. several kinds of - \ref range_req "ranges"), while other ones must always be implemented by the user. - - TBB defines a set of minimal requirements each concept must conform to. Here is - the list of different concepts hyperlinked to the corresponding requirements specifications: - - \subpage range_req - - \subpage parallel_do_body_req - - \subpage parallel_for_body_req - - \subpage parallel_reduce_body_req - - \subpage parallel_scan_body_req - - \subpage parallel_sort_iter_req -**/ - -// tbb_config.h should be included the first since it contains macro definitions used in other headers -#include "tbb_config.h" - -#if _MSC_VER >=1400 - #define __TBB_EXPORTED_FUNC __cdecl - #define __TBB_EXPORTED_METHOD __thiscall -#else - #define __TBB_EXPORTED_FUNC - #define __TBB_EXPORTED_METHOD -#endif - -#if __INTEL_COMPILER || _MSC_VER -#define __TBB_NOINLINE(decl) __declspec(noinline) decl -#elif __GNUC__ -#define __TBB_NOINLINE(decl) decl __attribute__ ((noinline)) -#else -#define __TBB_NOINLINE(decl) decl -#endif - -#if __TBB_NOEXCEPT_PRESENT -#define __TBB_NOEXCEPT(expression) noexcept(expression) -#else -#define __TBB_NOEXCEPT(expression) -#endif - -#include /* Need size_t and ptrdiff_t */ - -#if _MSC_VER - #define __TBB_tbb_windef_H - #include "internal/_tbb_windef.h" - #undef __TBB_tbb_windef_H -#endif -#if !defined(_MSC_VER) || _MSC_VER>=1600 - #include -#endif - -//! Type for an assertion handler -typedef void(*assertion_handler_type)( const char* filename, int line, const char* expression, const char * comment ); - -#if __TBBMALLOC_BUILD -namespace rml { namespace internal { - #define __TBB_ASSERT_RELEASE(predicate,message) ((predicate)?((void)0) : rml::internal::assertion_failure(__FILE__,__LINE__,#predicate,message)) -#else -namespace tbb { - #define __TBB_ASSERT_RELEASE(predicate,message) ((predicate)?((void)0) : tbb::assertion_failure(__FILE__,__LINE__,#predicate,message)) -#endif - - //! Set assertion handler and return previous value of it. - assertion_handler_type __TBB_EXPORTED_FUNC set_assertion_handler( assertion_handler_type new_handler ); - - //! Process an assertion failure. - /** Normally called from __TBB_ASSERT macro. - If assertion handler is null, print message for assertion failure and abort. - Otherwise call the assertion handler. */ - void __TBB_EXPORTED_FUNC assertion_failure( const char* filename, int line, const char* expression, const char* comment ); - -#if __TBBMALLOC_BUILD -}} // namespace rml::internal -#else -} // namespace tbb -#endif - -#if TBB_USE_ASSERT - - //! Assert that predicate is true. - /** If predicate is false, print assertion failure message. - If the comment argument is not NULL, it is printed as part of the failure message. - The comment argument has no other effect. */ - #define __TBB_ASSERT(predicate,message) __TBB_ASSERT_RELEASE(predicate,message) - - #define __TBB_ASSERT_EX __TBB_ASSERT - -#else /* !TBB_USE_ASSERT */ - - //! No-op version of __TBB_ASSERT. - #define __TBB_ASSERT(predicate,comment) ((void)0) - //! "Extended" version is useful to suppress warnings if a variable is only used with an assert - #define __TBB_ASSERT_EX(predicate,comment) ((void)(1 && (predicate))) - -#endif /* !TBB_USE_ASSERT */ - -//! The namespace tbb contains all components of the library. -namespace tbb { - - namespace internal { -#if _MSC_VER && _MSC_VER<1600 - typedef __int8 int8_t; - typedef __int16 int16_t; - typedef __int32 int32_t; - typedef __int64 int64_t; - typedef unsigned __int8 uint8_t; - typedef unsigned __int16 uint16_t; - typedef unsigned __int32 uint32_t; - typedef unsigned __int64 uint64_t; -#else /* Posix */ - using ::int8_t; - using ::int16_t; - using ::int32_t; - using ::int64_t; - using ::uint8_t; - using ::uint16_t; - using ::uint32_t; - using ::uint64_t; -#endif /* Posix */ - } // namespace internal - - using std::size_t; - using std::ptrdiff_t; - -//! The function returns the interface version of the TBB shared library being used. -/** - * The version it returns is determined at runtime, not at compile/link time. - * So it can be different than the value of TBB_INTERFACE_VERSION obtained at compile time. - */ -extern "C" int __TBB_EXPORTED_FUNC TBB_runtime_interface_version(); - -/** - * @cond INTERNAL - * @brief Identifiers declared inside namespace internal should never be used directly by client code. - */ -namespace internal { - -//! Compile-time constant that is upper bound on cache line/sector size. -/** It should be used only in situations where having a compile-time upper - bound is more useful than a run-time exact answer. - @ingroup memory_allocation */ -const size_t NFS_MaxLineSize = 128; - -/** Label for data that may be accessed from different threads, and that may eventually become wrapped - in a formal atomic type. - - Note that no problems have yet been observed relating to the definition currently being empty, - even if at least "volatile" would seem to be in order to avoid data sometimes temporarily hiding - in a register (although "volatile" as a "poor man's atomic" lacks several other features of a proper - atomic, some of which are now provided instead through specialized functions). - - Note that usage is intentionally compatible with a definition as qualifier "volatile", - both as a way to have the compiler help enforce use of the label and to quickly rule out - one potential issue. - - Note however that, with some architecture/compiler combinations, e.g. on IA-64 architecture, "volatile" - also has non-portable memory semantics that are needlessly expensive for "relaxed" operations. - - Note that this must only be applied to data that will not change bit patterns when cast to/from - an integral type of the same length; tbb::atomic must be used instead for, e.g., floating-point types. - - TODO: apply wherever relevant **/ -#define __TBB_atomic // intentionally empty, see above - -#if __TBB_OVERRIDE_PRESENT -#define __TBB_override override -#else -#define __TBB_override // formal comment only -#endif - -#if __TBB_CPP17_FALLTHROUGH_PRESENT -#define __TBB_fallthrough [[fallthrough]] -#elif __TBB_FALLTHROUGH_PRESENT -#define __TBB_fallthrough __attribute__ ((fallthrough)) -#else -#define __TBB_fallthrough -#endif - -template -struct padded_base : T { - char pad[S - R]; -}; -template struct padded_base : T {}; - -//! Pads type T to fill out to a multiple of cache line size. -template -struct padded : padded_base {}; - -//! Extended variant of the standard offsetof macro -/** The standard offsetof macro is not sufficient for TBB as it can be used for - POD-types only. The constant 0x1000 (not NULL) is necessary to appease GCC. **/ -#define __TBB_offsetof(class_name, member_name) \ - ((ptrdiff_t)&(reinterpret_cast(0x1000)->member_name) - 0x1000) - -//! Returns address of the object containing a member with the given name and address -#define __TBB_get_object_ref(class_name, member_name, member_addr) \ - (*reinterpret_cast((char*)member_addr - __TBB_offsetof(class_name, member_name))) - -//! Throws std::runtime_error with what() returning error_code description prefixed with aux_info -void __TBB_EXPORTED_FUNC handle_perror( int error_code, const char* aux_info ); - -#if TBB_USE_EXCEPTIONS - #define __TBB_TRY try - #define __TBB_CATCH(e) catch(e) - #define __TBB_THROW(e) throw e - #define __TBB_RETHROW() throw -#else /* !TBB_USE_EXCEPTIONS */ - inline bool __TBB_false() { return false; } - #define __TBB_TRY - #define __TBB_CATCH(e) if ( tbb::internal::__TBB_false() ) - #define __TBB_THROW(e) tbb::internal::suppress_unused_warning(e) - #define __TBB_RETHROW() ((void)0) -#endif /* !TBB_USE_EXCEPTIONS */ - -//! Report a runtime warning. -void __TBB_EXPORTED_FUNC runtime_warning( const char* format, ... ); - -#if TBB_USE_ASSERT -static void* const poisoned_ptr = reinterpret_cast(-1); - -//! Set p to invalid pointer value. -// Also works for regular (non-__TBB_atomic) pointers. -template -inline void poison_pointer( T* __TBB_atomic & p ) { p = reinterpret_cast(poisoned_ptr); } - -/** Expected to be used in assertions only, thus no empty form is defined. **/ -template -inline bool is_poisoned( T* p ) { return p == reinterpret_cast(poisoned_ptr); } -#else -template -inline void poison_pointer( T* __TBB_atomic & ) {/*do nothing*/} -#endif /* !TBB_USE_ASSERT */ - -//! Cast between unrelated pointer types. -/** This method should be used sparingly as a last resort for dealing with - situations that inherently break strict ISO C++ aliasing rules. */ -// T is a pointer type because it will be explicitly provided by the programmer as a template argument; -// U is a referent type to enable the compiler to check that "ptr" is a pointer, deducing U in the process. -template -inline T punned_cast( U* ptr ) { - uintptr_t x = reinterpret_cast(ptr); - return reinterpret_cast(x); -} - -#if __TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT - -//! Base class for types that should not be assigned. -class no_assign { -public: - void operator=( const no_assign& ) = delete; - no_assign( const no_assign& ) = default; - no_assign() = default; -}; - -//! Base class for types that should not be copied or assigned. -class no_copy: no_assign { -public: - no_copy( const no_copy& ) = delete; - no_copy() = default; -}; - -#else /*__TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT*/ - -//! Base class for types that should not be assigned. -class no_assign { - // Deny assignment - void operator=( const no_assign& ); -public: -#if __GNUC__ - //! Explicitly define default construction, because otherwise gcc issues gratuitous warning. - no_assign() {} -#endif /* __GNUC__ */ -}; - -//! Base class for types that should not be copied or assigned. -class no_copy: no_assign { - //! Deny copy construction - no_copy( const no_copy& ); -public: - //! Allow default construction - no_copy() {} -}; - -#endif /*__TBB_DEFAULTED_AND_DELETED_FUNC_PRESENT*/ - -#if TBB_DEPRECATED_MUTEX_COPYING -class mutex_copy_deprecated_and_disabled {}; -#else -// By default various implementations of mutexes are not copy constructible -// and not copy assignable. -class mutex_copy_deprecated_and_disabled : no_copy {}; -#endif - -//! A function to check if passed in pointer is aligned on a specific border -template -inline bool is_aligned(T* pointer, uintptr_t alignment) { - return 0==((uintptr_t)pointer & (alignment-1)); -} - -//! A function to check if passed integer is a power of 2 -template -inline bool is_power_of_two(integer_type arg) { - return arg && (0 == (arg & (arg - 1))); -} - -//! A function to compute arg modulo divisor where divisor is a power of 2. -template -inline argument_integer_type modulo_power_of_two(argument_integer_type arg, divisor_integer_type divisor) { - __TBB_ASSERT( is_power_of_two(divisor), "Divisor should be a power of two" ); - return (arg & (divisor - 1)); -} - - -//! A function to determine if arg is a power of 2 at least as big as another power of 2. -// i.e. for strictly positive i and j, with j being a power of 2, -// determines whether i==j< -inline bool is_power_of_two_at_least(argument_integer_type arg, power2_integer_type power2) { - __TBB_ASSERT( is_power_of_two(power2), "Divisor should be a power of two" ); - return 0 == (arg & (arg - power2)); -} - -//! Utility template function to prevent "unused" warnings by various compilers. -template void suppress_unused_warning( const T1& ) {} -template void suppress_unused_warning( const T1&, const T2& ) {} -template void suppress_unused_warning( const T1&, const T2&, const T3& ) {} - -// Struct to be used as a version tag for inline functions. -/** Version tag can be necessary to prevent loader on Linux from using the wrong - symbol in debug builds (when inline functions are compiled as out-of-line). **/ -struct version_tag_v3 {}; - -typedef version_tag_v3 version_tag; - -} // internal - -//! Dummy type that distinguishes splitting constructor from copy constructor. -/** - * See description of parallel_for and parallel_reduce for example usages. - * @ingroup algorithms - */ -class split { -}; - -//! Type enables transmission of splitting proportion from partitioners to range objects -/** - * In order to make use of such facility Range objects must implement - * splitting constructor with this type passed and initialize static - * constant boolean field 'is_splittable_in_proportion' with the value - * of 'true' - */ -class proportional_split: internal::no_assign { -public: - proportional_split(size_t _left = 1, size_t _right = 1) : my_left(_left), my_right(_right) { } - - size_t left() const { return my_left; } - size_t right() const { return my_right; } - - // used when range does not support proportional split - operator split() const { return split(); } - -#if __TBB_ENABLE_RANGE_FEEDBACK - void set_proportion(size_t _left, size_t _right) { - my_left = _left; - my_right = _right; - } -#endif -private: - size_t my_left, my_right; -}; - -} // tbb - -// Following is a set of classes and functions typically used in compile-time "metaprogramming". -// TODO: move all that to a separate header - -#if __TBB_CPP11_SMART_POINTERS_PRESENT -#include // for unique_ptr -#endif - -#if __TBB_CPP11_RVALUE_REF_PRESENT || __TBB_CPP11_DECLTYPE_PRESENT || _LIBCPP_VERSION -#include // for std::move, std::forward, std::declval -#endif - -namespace tbb { -namespace internal { - -#if __TBB_CPP11_SMART_POINTERS_PRESENT && __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT - template - std::unique_ptr make_unique(Args&&... args) { - return std::unique_ptr(new T(std::forward(args)...)); - } -#endif - -//! Class for determining type of std::allocator::value_type. -template -struct allocator_type { - typedef T value_type; -}; - -#if _MSC_VER -//! Microsoft std::allocator has non-standard extension that strips const from a type. -template -struct allocator_type { - typedef T value_type; -}; -#endif - -// Ad-hoc implementation of true_type & false_type -// Intended strictly for internal use! For public APIs (traits etc), use C++11 analogues. -template -struct bool_constant { - static /*constexpr*/ const bool value = v; -}; -typedef bool_constant true_type; -typedef bool_constant false_type; - -//! A template to select either 32-bit or 64-bit constant as compile time, depending on machine word size. -template -struct select_size_t_constant { - //Explicit cast is needed to avoid compiler warnings about possible truncation. - //The value of the right size, which is selected by ?:, is anyway not truncated or promoted. - static const size_t value = (size_t)((sizeof(size_t)==sizeof(u)) ? u : ull); -}; - -#if __TBB_CPP11_RVALUE_REF_PRESENT -using std::move; -using std::forward; -#elif defined(_LIBCPP_NAMESPACE) -// libc++ defines "pre-C++11 move and forward" similarly to ours; use it to avoid name conflicts in some cases. -using std::_LIBCPP_NAMESPACE::move; -using std::_LIBCPP_NAMESPACE::forward; -#else -// It is assumed that cv qualifiers, if any, are part of the deduced type. -template -T& move( T& x ) { return x; } -template -T& forward( T& x ) { return x; } -#endif /* __TBB_CPP11_RVALUE_REF_PRESENT */ - -// Helper macros to simplify writing templates working with both C++03 and C++11. -#if __TBB_CPP11_RVALUE_REF_PRESENT -#define __TBB_FORWARDING_REF(A) A&& -#else -// It is assumed that cv qualifiers, if any, are part of a deduced type. -// Thus this macro should not be used in public interfaces. -#define __TBB_FORWARDING_REF(A) A& -#endif -#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT -#define __TBB_PARAMETER_PACK ... -#define __TBB_PACK_EXPANSION(A) A... -#else -#define __TBB_PARAMETER_PACK -#define __TBB_PACK_EXPANSION(A) A -#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT */ - -#if __TBB_CPP11_DECLTYPE_PRESENT -#if __TBB_CPP11_DECLVAL_BROKEN -// Ad-hoc implementation of std::declval -template __TBB_FORWARDING_REF(T) declval() /*noexcept*/; -#else -using std::declval; -#endif -#endif - -template -struct STATIC_ASSERTION_FAILED; - -template <> -struct STATIC_ASSERTION_FAILED { enum {value=1};}; - -template<> -struct STATIC_ASSERTION_FAILED; //intentionally left undefined to cause compile time error - -//! @endcond -}} // namespace tbb::internal - -#if __TBB_STATIC_ASSERT_PRESENT -#define __TBB_STATIC_ASSERT(condition,msg) static_assert(condition,msg) -#else -//please note condition is intentionally inverted to get a bit more understandable error msg -#define __TBB_STATIC_ASSERT_IMPL1(condition,msg,line) \ - enum {static_assert_on_line_##line = tbb::internal::STATIC_ASSERTION_FAILED::value} - -#define __TBB_STATIC_ASSERT_IMPL(condition,msg,line) __TBB_STATIC_ASSERT_IMPL1(condition,msg,line) -//! Verify condition, at compile time -#define __TBB_STATIC_ASSERT(condition,msg) __TBB_STATIC_ASSERT_IMPL(condition,msg,__LINE__) -#endif - -#endif /* RC_INVOKED */ -#endif /* __TBB_tbb_stddef_H */ diff --git a/src/3rd party/tbb/tbb_thread.h b/src/3rd party/tbb/tbb_thread.h deleted file mode 100644 index 4558411e26..0000000000 --- a/src/3rd party/tbb/tbb_thread.h +++ /dev/null @@ -1,345 +0,0 @@ -/* - Copyright (c) 2005-2019 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -#include "internal/_deprecated_header_message_guard.h" - -#if !defined(__TBB_show_deprecation_message_tbb_thread_H) && defined(__TBB_show_deprecated_header_message) -#define __TBB_show_deprecation_message_tbb_thread_H -#pragma message("TBB Warning: tbb/tbb_thread.h is deprecated. For details, please see Deprecated Features appendix in the TBB reference manual.") -#endif - -#if defined(__TBB_show_deprecated_header_message) -#undef __TBB_show_deprecated_header_message -#endif - -#ifndef __TBB_tbb_thread_H -#define __TBB_tbb_thread_H - -#define __TBB_tbb_thread_H_include_area -#include "internal/_warning_suppress_enable_notice.h" - -#include "tbb_stddef.h" - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#define __TBB_NATIVE_THREAD_ROUTINE unsigned WINAPI -#define __TBB_NATIVE_THREAD_ROUTINE_PTR(r) unsigned (WINAPI* r)( void* ) -namespace tbb { namespace internal { -#if __TBB_WIN8UI_SUPPORT - typedef size_t thread_id_type; -#else // __TBB_WIN8UI_SUPPORT - typedef DWORD thread_id_type; -#endif // __TBB_WIN8UI_SUPPORT -}} //namespace tbb::internal -#else -#define __TBB_NATIVE_THREAD_ROUTINE void* -#define __TBB_NATIVE_THREAD_ROUTINE_PTR(r) void* (*r)( void* ) -#include -namespace tbb { namespace internal { - typedef pthread_t thread_id_type; -}} //namespace tbb::internal -#endif // _WIN32||_WIN64 - -#include "atomic.h" -#include "internal/_tbb_hash_compare_impl.h" -#include "tick_count.h" - -#include __TBB_STD_SWAP_HEADER -#include - -namespace tbb { - -namespace internal { - class tbb_thread_v3; -} - -inline void swap( internal::tbb_thread_v3& t1, internal::tbb_thread_v3& t2 ) __TBB_NOEXCEPT(true); - -namespace internal { - - //! Allocate a closure - void* __TBB_EXPORTED_FUNC allocate_closure_v3( size_t size ); - //! Free a closure allocated by allocate_closure_v3 - void __TBB_EXPORTED_FUNC free_closure_v3( void* ); - - struct thread_closure_base { - void* operator new( size_t size ) {return allocate_closure_v3(size);} - void operator delete( void* ptr ) {free_closure_v3(ptr);} - }; - - template struct thread_closure_0: thread_closure_base { - F function; - - static __TBB_NATIVE_THREAD_ROUTINE start_routine( void* c ) { - thread_closure_0 *self = static_cast(c); - self->function(); - delete self; - return 0; - } - thread_closure_0( const F& f ) : function(f) {} - }; - //! Structure used to pass user function with 1 argument to thread. - template struct thread_closure_1: thread_closure_base { - F function; - X arg1; - //! Routine passed to Windows's _beginthreadex by thread::internal_start() inside tbb.dll - static __TBB_NATIVE_THREAD_ROUTINE start_routine( void* c ) { - thread_closure_1 *self = static_cast(c); - self->function(self->arg1); - delete self; - return 0; - } - thread_closure_1( const F& f, const X& x ) : function(f), arg1(x) {} - }; - template struct thread_closure_2: thread_closure_base { - F function; - X arg1; - Y arg2; - //! Routine passed to Windows's _beginthreadex by thread::internal_start() inside tbb.dll - static __TBB_NATIVE_THREAD_ROUTINE start_routine( void* c ) { - thread_closure_2 *self = static_cast(c); - self->function(self->arg1, self->arg2); - delete self; - return 0; - } - thread_closure_2( const F& f, const X& x, const Y& y ) : function(f), arg1(x), arg2(y) {} - }; - - //! Versioned thread class. - class tbb_thread_v3 { -#if __TBB_IF_NO_COPY_CTOR_MOVE_SEMANTICS_BROKEN - // Workaround for a compiler bug: declaring the copy constructor as public - // enables use of the moving constructor. - // The definition is not provided in order to prohibit copying. - public: -#endif - tbb_thread_v3(const tbb_thread_v3&); // = delete; // Deny access - public: -#if _WIN32||_WIN64 - typedef HANDLE native_handle_type; -#else - typedef pthread_t native_handle_type; -#endif // _WIN32||_WIN64 - - class id; - //! Constructs a thread object that does not represent a thread of execution. - tbb_thread_v3() __TBB_NOEXCEPT(true) : my_handle(0) -#if _WIN32||_WIN64 - , my_thread_id(0) -#endif // _WIN32||_WIN64 - {} - - //! Constructs an object and executes f() in a new thread - template explicit tbb_thread_v3(F f) { - typedef internal::thread_closure_0 closure_type; - internal_start(closure_type::start_routine, new closure_type(f)); - } - //! Constructs an object and executes f(x) in a new thread - template tbb_thread_v3(F f, X x) { - typedef internal::thread_closure_1 closure_type; - internal_start(closure_type::start_routine, new closure_type(f,x)); - } - //! Constructs an object and executes f(x,y) in a new thread - template tbb_thread_v3(F f, X x, Y y) { - typedef internal::thread_closure_2 closure_type; - internal_start(closure_type::start_routine, new closure_type(f,x,y)); - } - -#if __TBB_CPP11_RVALUE_REF_PRESENT - tbb_thread_v3(tbb_thread_v3&& x) __TBB_NOEXCEPT(true) - : my_handle(x.my_handle) -#if _WIN32||_WIN64 - , my_thread_id(x.my_thread_id) -#endif - { - x.internal_wipe(); - } - tbb_thread_v3& operator=(tbb_thread_v3&& x) __TBB_NOEXCEPT(true) { - internal_move(x); - return *this; - } - private: - tbb_thread_v3& operator=(const tbb_thread_v3& x); // = delete; - public: -#else // __TBB_CPP11_RVALUE_REF_PRESENT - tbb_thread_v3& operator=(tbb_thread_v3& x) { - internal_move(x); - return *this; - } -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - - void swap( tbb_thread_v3& t ) __TBB_NOEXCEPT(true) {tbb::swap( *this, t );} - bool joinable() const __TBB_NOEXCEPT(true) {return my_handle!=0; } - //! The completion of the thread represented by *this happens before join() returns. - void __TBB_EXPORTED_METHOD join(); - //! When detach() returns, *this no longer represents the possibly continuing thread of execution. - void __TBB_EXPORTED_METHOD detach(); - ~tbb_thread_v3() {if( joinable() ) detach();} - inline id get_id() const __TBB_NOEXCEPT(true); - native_handle_type native_handle() { return my_handle; } - - //! The number of hardware thread contexts. - /** Before TBB 3.0 U4 this methods returned the number of logical CPU in - the system. Currently on Windows, Linux and FreeBSD it returns the - number of logical CPUs available to the current process in accordance - with its affinity mask. - - NOTE: The return value of this method never changes after its first - invocation. This means that changes in the process affinity mask that - took place after this method was first invoked will not affect the - number of worker threads in the TBB worker threads pool. **/ - static unsigned __TBB_EXPORTED_FUNC hardware_concurrency() __TBB_NOEXCEPT(true); - private: - native_handle_type my_handle; -#if _WIN32||_WIN64 - thread_id_type my_thread_id; -#endif // _WIN32||_WIN64 - - void internal_wipe() __TBB_NOEXCEPT(true) { - my_handle = 0; -#if _WIN32||_WIN64 - my_thread_id = 0; -#endif - } - void internal_move(tbb_thread_v3& x) __TBB_NOEXCEPT(true) { - if (joinable()) detach(); - my_handle = x.my_handle; -#if _WIN32||_WIN64 - my_thread_id = x.my_thread_id; -#endif // _WIN32||_WIN64 - x.internal_wipe(); - } - - /** Runs start_routine(closure) on another thread and sets my_handle to the handle of the created thread. */ - void __TBB_EXPORTED_METHOD internal_start( __TBB_NATIVE_THREAD_ROUTINE_PTR(start_routine), - void* closure ); - friend void __TBB_EXPORTED_FUNC move_v3( tbb_thread_v3& t1, tbb_thread_v3& t2 ); - friend void tbb::swap( tbb_thread_v3& t1, tbb_thread_v3& t2 ) __TBB_NOEXCEPT(true); - }; - - class tbb_thread_v3::id { - thread_id_type my_id; - id( thread_id_type id_ ) : my_id(id_) {} - - friend class tbb_thread_v3; - public: - id() __TBB_NOEXCEPT(true) : my_id(0) {} - - friend bool operator==( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - friend bool operator!=( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - friend bool operator<( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - friend bool operator<=( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - friend bool operator>( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - friend bool operator>=( tbb_thread_v3::id x, tbb_thread_v3::id y ) __TBB_NOEXCEPT(true); - - template - friend std::basic_ostream& - operator<< (std::basic_ostream &out, - tbb_thread_v3::id id) - { - out << id.my_id; - return out; - } - friend tbb_thread_v3::id __TBB_EXPORTED_FUNC thread_get_id_v3(); - - friend inline size_t tbb_hasher( const tbb_thread_v3::id& id ) { - __TBB_STATIC_ASSERT(sizeof(id.my_id) <= sizeof(size_t), "Implementation assumes that thread_id_type fits into machine word"); - return tbb::tbb_hasher(id.my_id); - } - - // A workaround for lack of tbb::atomic (which would require id to be POD in C++03). - friend id atomic_compare_and_swap(id& location, const id& value, const id& comparand){ - return as_atomic(location.my_id).compare_and_swap(value.my_id, comparand.my_id); - } - }; // tbb_thread_v3::id - - tbb_thread_v3::id tbb_thread_v3::get_id() const __TBB_NOEXCEPT(true) { -#if _WIN32||_WIN64 - return id(my_thread_id); -#else - return id(my_handle); -#endif // _WIN32||_WIN64 - } - - void __TBB_EXPORTED_FUNC move_v3( tbb_thread_v3& t1, tbb_thread_v3& t2 ); - tbb_thread_v3::id __TBB_EXPORTED_FUNC thread_get_id_v3(); - void __TBB_EXPORTED_FUNC thread_yield_v3(); - void __TBB_EXPORTED_FUNC thread_sleep_v3(const tick_count::interval_t &i); - - inline bool operator==(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id == y.my_id; - } - inline bool operator!=(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id != y.my_id; - } - inline bool operator<(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id < y.my_id; - } - inline bool operator<=(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id <= y.my_id; - } - inline bool operator>(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id > y.my_id; - } - inline bool operator>=(tbb_thread_v3::id x, tbb_thread_v3::id y) __TBB_NOEXCEPT(true) - { - return x.my_id >= y.my_id; - } - -} // namespace internal; - -//! Users reference thread class by name tbb_thread -__TBB_DEPRECATED_VERBOSE_MSG("tbb::thread is deprecated, use std::thread") typedef internal::tbb_thread_v3 tbb_thread; - -using internal::operator==; -using internal::operator!=; -using internal::operator<; -using internal::operator>; -using internal::operator<=; -using internal::operator>=; - -inline void move( tbb_thread& t1, tbb_thread& t2 ) { - internal::move_v3(t1, t2); -} - -inline void swap( internal::tbb_thread_v3& t1, internal::tbb_thread_v3& t2 ) __TBB_NOEXCEPT(true) { - std::swap(t1.my_handle, t2.my_handle); -#if _WIN32||_WIN64 - std::swap(t1.my_thread_id, t2.my_thread_id); -#endif /* _WIN32||_WIN64 */ -} - -namespace this_tbb_thread { - __TBB_DEPRECATED_VERBOSE inline tbb_thread::id get_id() { return internal::thread_get_id_v3(); } - //! Offers the operating system the opportunity to schedule another thread. - __TBB_DEPRECATED_VERBOSE inline void yield() { internal::thread_yield_v3(); } - //! The current thread blocks at least until the time specified. - __TBB_DEPRECATED_VERBOSE inline void sleep(const tick_count::interval_t &i) { - internal::thread_sleep_v3(i); - } -} // namespace this_tbb_thread - -} // namespace tbb - -#include "internal/_warning_suppress_disable_notice.h" -#undef __TBB_tbb_thread_H_include_area - -#endif /* __TBB_tbb_thread_H */ diff --git a/src/3rd party/tbb/tbbmalloc_proxy.h b/src/3rd party/tbb/tbbmalloc_proxy.h index d7b7235a0b..93eaa18e80 100644 --- a/src/3rd party/tbb/tbbmalloc_proxy.h +++ b/src/3rd party/tbb/tbbmalloc_proxy.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,52 +14,4 @@ limitations under the License. */ -/* -Replacing the standard memory allocation routines in Microsoft* C/C++ RTL -(malloc/free, global new/delete, etc.) with the TBB memory allocator. - -Include the following header to a source of any binary which is loaded during -application startup - -#include "tbb/tbbmalloc_proxy.h" - -or add following parameters to the linker options for the binary which is -loaded during application startup. It can be either exe-file or dll. - -For win32 -tbbmalloc_proxy.lib /INCLUDE:"___TBB_malloc_proxy" -win64 -tbbmalloc_proxy.lib /INCLUDE:"__TBB_malloc_proxy" -*/ - -#ifndef __TBB_tbbmalloc_proxy_H -#define __TBB_tbbmalloc_proxy_H - -#if _MSC_VER - -#ifdef _DEBUG - #pragma comment(lib, "tbbmalloc_proxy_debug.lib") -#else - #pragma comment(lib, "tbbmalloc_proxy.lib") -#endif - -#if defined(_WIN64) - #pragma comment(linker, "/include:__TBB_malloc_proxy") -#else - #pragma comment(linker, "/include:___TBB_malloc_proxy") -#endif - -#else -/* Primarily to support MinGW */ - -extern "C" void __TBB_malloc_proxy(); -struct __TBB_malloc_proxy_caller { - __TBB_malloc_proxy_caller() { __TBB_malloc_proxy(); } -} volatile __TBB_malloc_proxy_helper_object; - -#endif // _MSC_VER - -/* Public Windows API */ -extern "C" int TBB_malloc_replacement_log(char *** function_replacement_log_ptr); - -#endif //__TBB_tbbmalloc_proxy_H +#include "../oneapi/tbb/tbbmalloc_proxy.h" diff --git a/src/3rd party/tbb/tick_count.h b/src/3rd party/tbb/tick_count.h index a3c744ceb9..170074aebb 100644 --- a/src/3rd party/tbb/tick_count.h +++ b/src/3rd party/tbb/tick_count.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2005-2019 Intel Corporation + Copyright (c) 2005-2021 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,123 +14,4 @@ limitations under the License. */ -#ifndef __TBB_tick_count_H -#define __TBB_tick_count_H - -#include "tbb_stddef.h" - -#if _WIN32||_WIN64 -#include "machine/windows_api.h" -#elif __linux__ -#include -#else /* generic Unix */ -#include -#endif /* (choice of OS) */ - -namespace tbb { - -//! Absolute timestamp -/** @ingroup timing */ -class tick_count { -public: - //! Relative time interval. - class interval_t { - long long value; - explicit interval_t( long long value_ ) : value(value_) {} - public: - //! Construct a time interval representing zero time duration - interval_t() : value(0) {}; - - //! Construct a time interval representing sec seconds time duration - explicit interval_t( double sec ); - - //! Return the length of a time interval in seconds - double seconds() const; - - friend class tbb::tick_count; - - //! Extract the intervals from the tick_counts and subtract them. - friend interval_t operator-( const tick_count& t1, const tick_count& t0 ); - - //! Add two intervals. - friend interval_t operator+( const interval_t& i, const interval_t& j ) { - return interval_t(i.value+j.value); - } - - //! Subtract two intervals. - friend interval_t operator-( const interval_t& i, const interval_t& j ) { - return interval_t(i.value-j.value); - } - - //! Accumulation operator - interval_t& operator+=( const interval_t& i ) {value += i.value; return *this;} - - //! Subtraction operator - interval_t& operator-=( const interval_t& i ) {value -= i.value; return *this;} - private: - static long long ticks_per_second(){ -#if _WIN32||_WIN64 - LARGE_INTEGER qpfreq; - int rval = QueryPerformanceFrequency(&qpfreq); - __TBB_ASSERT_EX(rval, "QueryPerformanceFrequency returned zero"); - return static_cast(qpfreq.QuadPart); -#elif __linux__ - return static_cast(1E9); -#else /* generic Unix */ - return static_cast(1E6); -#endif /* (choice of OS) */ - } - }; - - //! Construct an absolute timestamp initialized to zero. - tick_count() : my_count(0) {}; - - //! Return current time. - static tick_count now(); - - //! Subtract two timestamps to get the time interval between - friend interval_t operator-( const tick_count& t1, const tick_count& t0 ); - - //! Return the resolution of the clock in seconds per tick. - static double resolution() { return 1.0 / interval_t::ticks_per_second(); } - -private: - long long my_count; -}; - -inline tick_count tick_count::now() { - tick_count result; -#if _WIN32||_WIN64 - LARGE_INTEGER qpcnt; - int rval = QueryPerformanceCounter(&qpcnt); - __TBB_ASSERT_EX(rval, "QueryPerformanceCounter failed"); - result.my_count = qpcnt.QuadPart; -#elif __linux__ - struct timespec ts; - int status = clock_gettime( CLOCK_REALTIME, &ts ); - __TBB_ASSERT_EX( status==0, "CLOCK_REALTIME not supported" ); - result.my_count = static_cast(1000000000UL)*static_cast(ts.tv_sec) + static_cast(ts.tv_nsec); -#else /* generic Unix */ - struct timeval tv; - int status = gettimeofday(&tv, NULL); - __TBB_ASSERT_EX( status==0, "gettimeofday failed" ); - result.my_count = static_cast(1000000)*static_cast(tv.tv_sec) + static_cast(tv.tv_usec); -#endif /*(choice of OS) */ - return result; -} - -inline tick_count::interval_t::interval_t( double sec ) { - value = static_cast(sec*interval_t::ticks_per_second()); -} - -inline tick_count::interval_t operator-( const tick_count& t1, const tick_count& t0 ) { - return tick_count::interval_t( t1.my_count-t0.my_count ); -} - -inline double tick_count::interval_t::seconds() const { - return value*tick_count::resolution(); -} - -} // namespace tbb - -#endif /* __TBB_tick_count_H */ +#include "../oneapi/tbb/tick_count.h" diff --git a/src/3rd party/tbb/version.h b/src/3rd party/tbb/version.h new file mode 100644 index 0000000000..cd13a83a15 --- /dev/null +++ b/src/3rd party/tbb/version.h @@ -0,0 +1,17 @@ +/* + Copyright (c) 2005-2021 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "../oneapi/tbb/version.h"