diff --git a/cpp/src/arrow/acero/partition_util.h b/cpp/src/arrow/acero/partition_util.h
index d02e9cb03f08..52cc47bb8a99 100644
--- a/cpp/src/arrow/acero/partition_util.h
+++ b/cpp/src/arrow/acero/partition_util.h
@@ -22,6 +22,7 @@
 #include <cstdint>
 #include <functional>
 #include <random>
+
 #include "arrow/acero/util.h"
 #include "arrow/buffer.h"
 #include "arrow/util/pcg_random.h"
diff --git a/cpp/src/arrow/array/data.cc b/cpp/src/arrow/array/data.cc
index 2e55668fb96e..b2fe52f9bbe5 100644
--- a/cpp/src/arrow/array/data.cc
+++ b/cpp/src/arrow/array/data.cc
@@ -34,7 +34,7 @@
 #include "arrow/type_traits.h"
 #include "arrow/util/binary_view_util.h"
 #include "arrow/util/bitmap_ops.h"
-#include "arrow/util/dict_util.h"
+#include "arrow/util/dict_util_internal.h"
 #include "arrow/util/logging_internal.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/range.h"
diff --git a/cpp/src/arrow/array/util.cc b/cpp/src/arrow/array/util.cc
index 3180b66a1f76..8bc8a20c5506 100644
--- a/cpp/src/arrow/array/util.cc
+++ b/cpp/src/arrow/array/util.cc
@@ -43,7 +43,7 @@
 #include "arrow/util/decimal.h"
 #include "arrow/util/endian.h"
 #include "arrow/util/logging_internal.h"
-#include "arrow/util/sort.h"
+#include "arrow/util/sort_internal.h"
 #include "arrow/util/span.h"
 #include "arrow/visit_data_inline.h"
 #include "arrow/visit_type_inline.h"
diff --git a/cpp/src/arrow/array/validate.cc b/cpp/src/arrow/array/validate.cc
index 3c9148ebb294..bd0d00126d51 100644
--- a/cpp/src/arrow/array/validate.cc
+++ b/cpp/src/arrow/array/validate.cc
@@ -30,7 +30,7 @@
 #include "arrow/util/int_util_overflow.h"
 #include "arrow/util/logging_internal.h"
 #include "arrow/util/ree_util.h"
-#include "arrow/util/sort.h"
+#include "arrow/util/sort_internal.h"
 #include "arrow/util/string.h"
 #include "arrow/util/unreachable.h"
 #include "arrow/util/utf8.h"
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 3b64a8fd09fb..0c5b654232e0 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -48,7 +48,7 @@
 #include "arrow/util/key_value_metadata.h"
 #include "arrow/util/logging_internal.h"
 #include "arrow/util/macros.h"
-#include "arrow/util/memory.h"
+#include "arrow/util/memory_internal.h"
 #include "arrow/util/ree_util.h"
 #include "arrow/visit_scalar_inline.h"
 #include "arrow/visit_type_inline.h"
diff --git a/cpp/src/arrow/compute/kernels/aggregate_tdigest.cc b/cpp/src/arrow/compute/kernels/aggregate_tdigest.cc
index 83d01091b3c8..10fa00689da3 100644
--- a/cpp/src/arrow/compute/kernels/aggregate_tdigest.cc
+++ b/cpp/src/arrow/compute/kernels/aggregate_tdigest.cc
@@ -19,7 +19,7 @@
 #include "arrow/compute/kernels/aggregate_internal.h"
 #include "arrow/compute/kernels/common_internal.h"
 #include "arrow/util/bit_run_reader.h"
-#include "arrow/util/tdigest.h"
+#include "arrow/util/tdigest_internal.h"
 
 namespace arrow {
 namespace compute {
diff --git a/cpp/src/arrow/compute/kernels/hash_aggregate_numeric.cc b/cpp/src/arrow/compute/kernels/hash_aggregate_numeric.cc
index 4a318942af62..6e0652efe693 100644
--- a/cpp/src/arrow/compute/kernels/hash_aggregate_numeric.cc
+++ b/cpp/src/arrow/compute/kernels/hash_aggregate_numeric.cc
@@ -32,7 +32,7 @@
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/int128_internal.h"
 #include "arrow/util/span.h"
-#include "arrow/util/tdigest.h"
+#include "arrow/util/tdigest_internal.h"
 #include "arrow/visit_type_inline.h"
 
 namespace arrow::compute::internal {
diff --git a/cpp/src/arrow/csv/converter.cc b/cpp/src/arrow/csv/converter.cc
index 3e991126d1f7..ec31d4b1ceb4 100644
--- a/cpp/src/arrow/csv/converter.cc
+++ b/cpp/src/arrow/csv/converter.cc
@@ -36,7 +36,7 @@
 #include "arrow/type_traits.h"
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/decimal.h"
-#include "arrow/util/trie.h"
+#include "arrow/util/trie_internal.h"
 #include "arrow/util/utf8_internal.h"
 #include "arrow/util/value_parsing.h"  // IWYU pragma: keep
 
diff --git a/cpp/src/arrow/dataset/dataset_writer.cc b/cpp/src/arrow/dataset/dataset_writer.cc
index 7b454b097578..43895374aa75 100644
--- a/cpp/src/arrow/dataset/dataset_writer.cc
+++ b/cpp/src/arrow/dataset/dataset_writer.cc
@@ -29,7 +29,7 @@
 #include "arrow/table.h"
 #include "arrow/util/future.h"
 #include "arrow/util/logging_internal.h"
-#include "arrow/util/map.h"
+#include "arrow/util/map_internal.h"
 #include "arrow/util/string.h"
 #include "arrow/util/tracing_internal.h"
 
diff --git a/cpp/src/arrow/dataset/file_base.cc b/cpp/src/arrow/dataset/file_base.cc
index 61f7c5e732ad..b03818314c02 100644
--- a/cpp/src/arrow/dataset/file_base.cc
+++ b/cpp/src/arrow/dataset/file_base.cc
@@ -46,7 +46,7 @@
 #include "arrow/util/iterator.h"
 #include "arrow/util/logging_internal.h"
 #include "arrow/util/macros.h"
-#include "arrow/util/map.h"
+#include "arrow/util/map_internal.h"
 #include "arrow/util/string.h"
 #include "arrow/util/task_group.h"
 #include "arrow/util/tracing_internal.h"
diff --git a/cpp/src/arrow/datum.cc b/cpp/src/arrow/datum.cc
index d7125d1f6c93..399007809863 100644
--- a/cpp/src/arrow/datum.cc
+++ b/cpp/src/arrow/datum.cc
@@ -31,7 +31,7 @@
 #include "arrow/table.h"
 #include "arrow/util/byte_size.h"
 #include "arrow/util/logging_internal.h"
-#include "arrow/util/memory.h"
+#include "arrow/util/memory_internal.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/extension/fixed_shape_tensor.cc b/cpp/src/arrow/extension/fixed_shape_tensor.cc
index 1b195e4c9f5e..d8fed85b1ed0 100644
--- a/cpp/src/arrow/extension/fixed_shape_tensor.cc
+++ b/cpp/src/arrow/extension/fixed_shape_tensor.cc
@@ -28,8 +28,8 @@
 #include "arrow/tensor.h"
 #include "arrow/util/int_util_overflow.h"
 #include "arrow/util/logging_internal.h"
-#include "arrow/util/print.h"
-#include "arrow/util/sort.h"
+#include "arrow/util/print_internal.h"
+#include "arrow/util/sort_internal.h"
 #include "arrow/util/string.h"
 
 #include <rapidjson/document.h>
diff --git a/cpp/src/arrow/extension/fixed_shape_tensor_test.cc b/cpp/src/arrow/extension/fixed_shape_tensor_test.cc
index 51aea4b25fdd..cfc126569660 100644
--- a/cpp/src/arrow/extension/fixed_shape_tensor_test.cc
+++ b/cpp/src/arrow/extension/fixed_shape_tensor_test.cc
@@ -28,7 +28,7 @@
 #include "arrow/tensor.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/util/key_value_metadata.h"
-#include "arrow/util/sort.h"
+#include "arrow/util/sort_internal.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/extension/tensor_internal.h b/cpp/src/arrow/extension/tensor_internal.h
index 069880cb17c8..ed5a7ffc790a 100644
--- a/cpp/src/arrow/extension/tensor_internal.h
+++ b/cpp/src/arrow/extension/tensor_internal.h
@@ -21,7 +21,7 @@
 #include <vector>
 
 #include "arrow/status.h"
-#include "arrow/util/print.h"
+#include "arrow/util/print_internal.h"
 
 namespace arrow::internal {
 
diff --git a/cpp/src/arrow/filesystem/s3_internal.h b/cpp/src/arrow/filesystem/s3_internal.h
index 772387e5fb66..f408b18ebd36 100644
--- a/cpp/src/arrow/filesystem/s3_internal.h
+++ b/cpp/src/arrow/filesystem/s3_internal.h
@@ -37,7 +37,7 @@
 #include "arrow/status.h"
 #include "arrow/util/base64.h"
 #include "arrow/util/logging.h"
-#include "arrow/util/print.h"
+#include "arrow/util/print_internal.h"
 #include "arrow/util/string.h"
 
 #ifndef ARROW_AWS_SDK_VERSION_CHECK
diff --git a/cpp/src/arrow/filesystem/test_util.h b/cpp/src/arrow/filesystem/test_util.h
index 3a643b7e9f08..3217cc8ca36a 100644
--- a/cpp/src/arrow/filesystem/test_util.h
+++ b/cpp/src/arrow/filesystem/test_util.h
@@ -25,7 +25,7 @@
 #include "arrow/filesystem/filesystem.h"
 #include "arrow/filesystem/mockfs.h"
 #include "arrow/testing/visibility.h"
-#include "arrow/util/counting_semaphore.h"
+#include "arrow/util/counting_semaphore_internal.h"
 
 namespace arrow {
 namespace fs {
diff --git a/cpp/src/arrow/flight/flight_benchmark.cc b/cpp/src/arrow/flight/flight_benchmark.cc
index 49e54d98f661..aa6e16820efb 100644
--- a/cpp/src/arrow/flight/flight_benchmark.cc
+++ b/cpp/src/arrow/flight/flight_benchmark.cc
@@ -31,8 +31,8 @@
 #include "arrow/testing/gtest_util.h"
 #include "arrow/util/compression.h"
 #include "arrow/util/config.h"
-#include "arrow/util/stopwatch.h"
-#include "arrow/util/tdigest.h"
+#include "arrow/util/stopwatch_internal.h"
+#include "arrow/util/tdigest_internal.h"
 #include "arrow/util/thread_pool.h"
 
 #include "arrow/flight/api.h"
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index 1ae03aeb1434..d7b118b39827 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -31,7 +31,7 @@
 #include "arrow/util/io_util.h"
 #include "arrow/util/logging_internal.h"
 #include "arrow/util/macros.h"
-#include "arrow/util/memory.h"
+#include "arrow/util/memory_internal.h"
 
 namespace arrow {
 namespace io {
diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc
index dbc32489937d..53f856d8012e 100644
--- a/cpp/src/arrow/json/parser.cc
+++ b/cpp/src/arrow/json/parser.cc
@@ -35,10 +35,10 @@
 #include "arrow/array/builder_binary.h"
 #include "arrow/buffer_builder.h"
 #include "arrow/type.h"
-#include "arrow/util/bitset_stack.h"
+#include "arrow/util/bitset_stack_internal.h"
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/logging_internal.h"
-#include "arrow/util/trie.h"
+#include "arrow/util/trie_internal.h"
 #include "arrow/visit_type_inline.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/sparse_tensor_test.cc b/cpp/src/arrow/sparse_tensor_test.cc
index 73477fef4a4c..c9c28a11b1b3 100644
--- a/cpp/src/arrow/sparse_tensor_test.cc
+++ b/cpp/src/arrow/sparse_tensor_test.cc
@@ -33,7 +33,7 @@
 #include "arrow/testing/util.h"
 #include "arrow/type.h"
 #include "arrow/util/logging_internal.h"
-#include "arrow/util/sort.h"
+#include "arrow/util/sort_internal.h"
 
 namespace arrow {
 
diff --git a/cpp/src/arrow/tensor/csf_converter.cc b/cpp/src/arrow/tensor/csf_converter.cc
index 2d925ddbbb01..f6470e16b781 100644
--- a/cpp/src/arrow/tensor/csf_converter.cc
+++ b/cpp/src/arrow/tensor/csf_converter.cc
@@ -29,7 +29,7 @@
 #include "arrow/status.h"
 #include "arrow/type.h"
 #include "arrow/util/checked_cast.h"
-#include "arrow/util/sort.h"
+#include "arrow/util/sort_internal.h"
 #include "arrow/visit_type_inline.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/util/bit_stream_utils_internal.h b/cpp/src/arrow/util/bit_stream_utils_internal.h
index d59e88a3449d..9d67c278bcc3 100644
--- a/cpp/src/arrow/util/bit_stream_utils_internal.h
+++ b/cpp/src/arrow/util/bit_stream_utils_internal.h
@@ -24,7 +24,7 @@
 #include <cstring>
 
 #include "arrow/util/bit_util.h"
-#include "arrow/util/bpacking.h"
+#include "arrow/util/bpacking_internal.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/ubsan.h"
diff --git a/cpp/src/arrow/util/bit_util_test.cc b/cpp/src/arrow/util/bit_util_test.cc
index 02f583e0110b..fcaeb49261d1 100644
--- a/cpp/src/arrow/util/bit_util_test.cc
+++ b/cpp/src/arrow/util/bit_util_test.cc
@@ -50,7 +50,7 @@
 #include "arrow/util/bitmap_reader.h"
 #include "arrow/util/bitmap_visit.h"
 #include "arrow/util/bitmap_writer.h"
-#include "arrow/util/bitset_stack.h"
+#include "arrow/util/bitset_stack_internal.h"
 #include "arrow/util/endian.h"
 #include "arrow/util/ubsan.h"
 
diff --git a/cpp/src/arrow/util/bitset_stack.h b/cpp/src/arrow/util/bitset_stack_internal.h
similarity index 100%
rename from cpp/src/arrow/util/bitset_stack.h
rename to cpp/src/arrow/util/bitset_stack_internal.h
diff --git a/cpp/src/arrow/util/bpacking.cc b/cpp/src/arrow/util/bpacking.cc
index 56ddd3762931..326dd050fe10 100644
--- a/cpp/src/arrow/util/bpacking.cc
+++ b/cpp/src/arrow/util/bpacking.cc
@@ -15,22 +15,22 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/bpacking.h"
+#include "arrow/util/bpacking_internal.h"
 
-#include "arrow/util/bpacking64_default.h"
-#include "arrow/util/bpacking_default.h"
+#include "arrow/util/bpacking64_default_internal.h"
+#include "arrow/util/bpacking_default_internal.h"
 #include "arrow/util/cpu_info.h"
-#include "arrow/util/dispatch.h"
+#include "arrow/util/dispatch_internal.h"
 #include "arrow/util/logging_internal.h"
 
 #if defined(ARROW_HAVE_RUNTIME_AVX2)
-#  include "arrow/util/bpacking_avx2.h"
+#  include "arrow/util/bpacking_avx2_internal.h"
 #endif
 #if defined(ARROW_HAVE_RUNTIME_AVX512)
-#  include "arrow/util/bpacking_avx512.h"
+#  include "arrow/util/bpacking_avx512_internal.h"
 #endif
 #if defined(ARROW_HAVE_NEON)
-#  include "arrow/util/bpacking_neon.h"
+#  include "arrow/util/bpacking_neon_internal.h"
 #endif
 
 namespace arrow {
diff --git a/cpp/src/arrow/util/bpacking64_codegen.py b/cpp/src/arrow/util/bpacking64_codegen.py
index 88dc93ca6579..22135fcbb235 100644
--- a/cpp/src/arrow/util/bpacking64_codegen.py
+++ b/cpp/src/arrow/util/bpacking64_codegen.py
@@ -21,7 +21,7 @@
 # https://github.com/lemire/FrameOfReference/blob/146948b6058a976bc7767262ad3a2ce201486b93/scripts/turbopacking64.py
 
 # Usage:
-#   python bpacking64_codegen.py > bpacking64_default.h
+#   python bpacking64_codegen.py > bpacking64_default_internal.h
 
 def howmany(bit):
     """ how many values are we going to pack? """
diff --git a/cpp/src/arrow/util/bpacking64_default.h b/cpp/src/arrow/util/bpacking64_default_internal.h
similarity index 100%
rename from cpp/src/arrow/util/bpacking64_default.h
rename to cpp/src/arrow/util/bpacking64_default_internal.h
diff --git a/cpp/src/arrow/util/bpacking_avx2.cc b/cpp/src/arrow/util/bpacking_avx2.cc
index 9105aaa2af41..02510a07b9f5 100644
--- a/cpp/src/arrow/util/bpacking_avx2.cc
+++ b/cpp/src/arrow/util/bpacking_avx2.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/bpacking_avx2.h"
+#include "arrow/util/bpacking_avx2_internal.h"
 #include "arrow/util/bpacking_simd256_generated_internal.h"
 #include "arrow/util/bpacking_simd_internal.h"
 
diff --git a/cpp/src/arrow/util/bpacking_avx2.h b/cpp/src/arrow/util/bpacking_avx2_internal.h
similarity index 100%
rename from cpp/src/arrow/util/bpacking_avx2.h
rename to cpp/src/arrow/util/bpacking_avx2_internal.h
diff --git a/cpp/src/arrow/util/bpacking_avx512.cc b/cpp/src/arrow/util/bpacking_avx512.cc
index 3570bcc352b1..6272ef1cde81 100644
--- a/cpp/src/arrow/util/bpacking_avx512.cc
+++ b/cpp/src/arrow/util/bpacking_avx512.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/bpacking_avx512.h"
+#include "arrow/util/bpacking_avx512_internal.h"
 #include "arrow/util/bpacking_simd512_generated_internal.h"
 #include "arrow/util/bpacking_simd_internal.h"
 
diff --git a/cpp/src/arrow/util/bpacking_avx512.h b/cpp/src/arrow/util/bpacking_avx512_internal.h
similarity index 100%
rename from cpp/src/arrow/util/bpacking_avx512.h
rename to cpp/src/arrow/util/bpacking_avx512_internal.h
diff --git a/cpp/src/arrow/util/bpacking_default.h b/cpp/src/arrow/util/bpacking_default_internal.h
similarity index 100%
rename from cpp/src/arrow/util/bpacking_default.h
rename to cpp/src/arrow/util/bpacking_default_internal.h
diff --git a/cpp/src/arrow/util/bpacking.h b/cpp/src/arrow/util/bpacking_internal.h
similarity index 100%
rename from cpp/src/arrow/util/bpacking.h
rename to cpp/src/arrow/util/bpacking_internal.h
diff --git a/cpp/src/arrow/util/bpacking_neon.cc b/cpp/src/arrow/util/bpacking_neon.cc
index 3ab6de75f4c6..72b520e8cf16 100644
--- a/cpp/src/arrow/util/bpacking_neon.cc
+++ b/cpp/src/arrow/util/bpacking_neon.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/bpacking_neon.h"
+#include "arrow/util/bpacking_neon_internal.h"
 #include "arrow/util/bpacking_simd128_generated_internal.h"
 #include "arrow/util/bpacking_simd_internal.h"
 
diff --git a/cpp/src/arrow/util/bpacking_neon.h b/cpp/src/arrow/util/bpacking_neon_internal.h
similarity index 100%
rename from cpp/src/arrow/util/bpacking_neon.h
rename to cpp/src/arrow/util/bpacking_neon_internal.h
diff --git a/cpp/src/arrow/util/bpacking_simd128_generated_internal.h b/cpp/src/arrow/util/bpacking_simd128_generated_internal.h
index 4b2c97c0a7dd..5beecad42103 100644
--- a/cpp/src/arrow/util/bpacking_simd128_generated_internal.h
+++ b/cpp/src/arrow/util/bpacking_simd128_generated_internal.h
@@ -24,7 +24,7 @@
 
 #include <xsimd/xsimd.hpp>
 
-#include "arrow/util/dispatch.h"
+#include "arrow/util/dispatch_internal.h"
 #include "arrow/util/ubsan.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/util/bpacking_simd256_generated_internal.h b/cpp/src/arrow/util/bpacking_simd256_generated_internal.h
index 8b1756d3fc19..3dccb1745f74 100644
--- a/cpp/src/arrow/util/bpacking_simd256_generated_internal.h
+++ b/cpp/src/arrow/util/bpacking_simd256_generated_internal.h
@@ -24,7 +24,7 @@
 
 #include <xsimd/xsimd.hpp>
 
-#include "arrow/util/dispatch.h"
+#include "arrow/util/dispatch_internal.h"
 #include "arrow/util/ubsan.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/util/bpacking_simd512_generated_internal.h b/cpp/src/arrow/util/bpacking_simd512_generated_internal.h
index deeb423353a5..4f2aeaeeb4b0 100644
--- a/cpp/src/arrow/util/bpacking_simd512_generated_internal.h
+++ b/cpp/src/arrow/util/bpacking_simd512_generated_internal.h
@@ -24,7 +24,7 @@
 
 #include <xsimd/xsimd.hpp>
 
-#include "arrow/util/dispatch.h"
+#include "arrow/util/dispatch_internal.h"
 #include "arrow/util/ubsan.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/util/bpacking_simd_codegen.py b/cpp/src/arrow/util/bpacking_simd_codegen.py
index 581a19a53e58..9464908c0217 100755
--- a/cpp/src/arrow/util/bpacking_simd_codegen.py
+++ b/cpp/src/arrow/util/bpacking_simd_codegen.py
@@ -164,7 +164,7 @@ def main(simd_width):
 
         #include <xsimd/xsimd.hpp>
 
-        #include "arrow/util/dispatch.h"
+        #include "arrow/util/dispatch_internal.h"
         #include "arrow/util/ubsan.h"
 
         namespace arrow {{
diff --git a/cpp/src/arrow/util/bpacking_simd_internal.h b/cpp/src/arrow/util/bpacking_simd_internal.h
index fa5a6689a563..98e192e7cb0b 100644
--- a/cpp/src/arrow/util/bpacking_simd_internal.h
+++ b/cpp/src/arrow/util/bpacking_simd_internal.h
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/dispatch.h"
+#include "arrow/util/dispatch_internal.h"
 #include "arrow/util/logging.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/util/counting_semaphore.cc b/cpp/src/arrow/util/counting_semaphore.cc
index b3106a6f824d..de9750ba9eed 100644
--- a/cpp/src/arrow/util/counting_semaphore.cc
+++ b/cpp/src/arrow/util/counting_semaphore.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/counting_semaphore.h"
+#include "arrow/util/counting_semaphore_internal.h"
 
 #include <chrono>
 #include <condition_variable>
diff --git a/cpp/src/arrow/util/counting_semaphore.h b/cpp/src/arrow/util/counting_semaphore_internal.h
similarity index 100%
rename from cpp/src/arrow/util/counting_semaphore.h
rename to cpp/src/arrow/util/counting_semaphore_internal.h
diff --git a/cpp/src/arrow/util/counting_semaphore_test.cc b/cpp/src/arrow/util/counting_semaphore_test.cc
index 4de11ce852a0..6635b1ddd7d8 100644
--- a/cpp/src/arrow/util/counting_semaphore_test.cc
+++ b/cpp/src/arrow/util/counting_semaphore_test.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/counting_semaphore.h"
+#include "arrow/util/counting_semaphore_internal.h"
 
 #include <atomic>
 #include <thread>
diff --git a/cpp/src/arrow/util/delimiting.cc b/cpp/src/arrow/util/delimiting.cc
index 4794293e0b4d..0bc1b45f6bf7 100644
--- a/cpp/src/arrow/util/delimiting.cc
+++ b/cpp/src/arrow/util/delimiting.cc
@@ -16,6 +16,7 @@
 // under the License.
 
 #include "arrow/util/delimiting.h"
+
 #include "arrow/buffer.h"
 #include "arrow/util/logging_internal.h"
 
diff --git a/cpp/src/arrow/util/dict_util.cc b/cpp/src/arrow/util/dict_util.cc
index feab2324a402..c93517140ca3 100644
--- a/cpp/src/arrow/util/dict_util.cc
+++ b/cpp/src/arrow/util/dict_util.cc
@@ -15,7 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/dict_util.h"
+#include "arrow/util/dict_util_internal.h"
+
 #include "arrow/array/array_dict.h"
 #include "arrow/util/bit_util.h"
 #include "arrow/util/checked_cast.h"
diff --git a/cpp/src/arrow/util/dict_util.h b/cpp/src/arrow/util/dict_util_internal.h
similarity index 100%
rename from cpp/src/arrow/util/dict_util.h
rename to cpp/src/arrow/util/dict_util_internal.h
diff --git a/cpp/src/arrow/util/dispatch.h b/cpp/src/arrow/util/dispatch_internal.h
similarity index 100%
rename from cpp/src/arrow/util/dispatch.h
rename to cpp/src/arrow/util/dispatch_internal.h
diff --git a/cpp/src/arrow/util/double_conversion.h b/cpp/src/arrow/util/double_conversion_internal.h
similarity index 100%
rename from cpp/src/arrow/util/double_conversion.h
rename to cpp/src/arrow/util/double_conversion_internal.h
diff --git a/cpp/src/arrow/util/formatting.cc b/cpp/src/arrow/util/formatting.cc
index 97567d863211..58dadd0b11e5 100644
--- a/cpp/src/arrow/util/formatting.cc
+++ b/cpp/src/arrow/util/formatting.cc
@@ -17,7 +17,7 @@
 
 #include "arrow/util/formatting.h"
 #include "arrow/util/config.h"
-#include "arrow/util/double_conversion.h"
+#include "arrow/util/double_conversion_internal.h"
 #include "arrow/util/float16.h"
 #include "arrow/util/logging_internal.h"
 
diff --git a/cpp/src/arrow/util/formatting.h b/cpp/src/arrow/util/formatting.h
index f2e3622ce60d..844b6fb91a8d 100644
--- a/cpp/src/arrow/util/formatting.h
+++ b/cpp/src/arrow/util/formatting.h
@@ -30,9 +30,8 @@
 #include <utility>
 
 #include "arrow/status.h"
-#include "arrow/type.h"
+#include "arrow/type_fwd.h"
 #include "arrow/type_traits.h"
-#include "arrow/util/double_conversion.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/string.h"
 #include "arrow/util/time.h"
diff --git a/cpp/src/arrow/util/key_value_metadata.cc b/cpp/src/arrow/util/key_value_metadata.cc
index 4390a4cb7950..48e02c612020 100644
--- a/cpp/src/arrow/util/key_value_metadata.cc
+++ b/cpp/src/arrow/util/key_value_metadata.cc
@@ -30,7 +30,7 @@
 #include "arrow/status.h"
 #include "arrow/util/key_value_metadata.h"
 #include "arrow/util/logging_internal.h"
-#include "arrow/util/sort.h"
+#include "arrow/util/sort_internal.h"
 
 using std::size_t;
 
diff --git a/cpp/src/arrow/util/map.h b/cpp/src/arrow/util/map_internal.h
similarity index 100%
rename from cpp/src/arrow/util/map.h
rename to cpp/src/arrow/util/map_internal.h
diff --git a/cpp/src/arrow/util/memory.cc b/cpp/src/arrow/util/memory.cc
index e91009d5860b..20f5ca7aae85 100644
--- a/cpp/src/arrow/util/memory.cc
+++ b/cpp/src/arrow/util/memory.cc
@@ -18,7 +18,7 @@
 #include <vector>
 
 #include "arrow/util/logging.h"
-#include "arrow/util/memory.h"
+#include "arrow/util/memory_internal.h"
 #include "arrow/util/thread_pool.h"
 
 namespace arrow {
diff --git a/cpp/src/arrow/util/memory.h b/cpp/src/arrow/util/memory_internal.h
similarity index 100%
rename from cpp/src/arrow/util/memory.h
rename to cpp/src/arrow/util/memory_internal.h
diff --git a/cpp/src/arrow/util/meson.build b/cpp/src/arrow/util/meson.build
index c398a5998421..584ea3ffe4c9 100644
--- a/cpp/src/arrow/util/meson.build
+++ b/cpp/src/arrow/util/meson.build
@@ -117,14 +117,7 @@ install_headers(
         'bitmap_visit.h',
         'bitmap_writer.h',
         'bit_run_reader.h',
-        'bitset_stack.h',
         'bit_util.h',
-        'bpacking64_default.h',
-        'bpacking_avx2.h',
-        'bpacking_avx512.h',
-        'bpacking_default.h',
-        'bpacking.h',
-        'bpacking_neon.h',
         'byte_size.h',
         'cancel.h',
         'checked_cast.h',
@@ -132,15 +125,11 @@ install_headers(
         'compression.h',
         'concurrent_map.h',
         'converter.h',
-        'counting_semaphore.h',
         'cpu_info.h',
         'crc32.h',
         'debug.h',
         'decimal.h',
         'delimiting.h',
-        'dict_util.h',
-        'dispatch.h',
-        'double_conversion.h',
         'endian.h',
         'float16.h',
         'formatting.h',
@@ -158,14 +147,11 @@ install_headers(
         'logger.h',
         'logging.h',
         'macros.h',
-        'map.h',
         'math_constants.h',
-        'memory.h',
         'mutex.h',
         'parallel.h',
         'pcg_random.h',
         'prefetch.h',
-        'print.h',
         'queue.h',
         'range.h',
         'ree_util.h',
@@ -173,19 +159,14 @@ install_headers(
         'rows_to_batches.h',
         'simd.h',
         'small_vector.h',
-        'sort.h',
-        'spaced.h',
         'span.h',
-        'stopwatch.h',
         'string_builder.h',
         'string.h',
         'task_group.h',
-        'tdigest.h',
         'test_common.h',
         'thread_pool.h',
         'time.h',
         'tracing.h',
-        'trie.h',
         'type_fwd.h',
         'type_traits.h',
         'ubsan.h',
diff --git a/cpp/src/arrow/util/print.h b/cpp/src/arrow/util/print_internal.h
similarity index 100%
rename from cpp/src/arrow/util/print.h
rename to cpp/src/arrow/util/print_internal.h
diff --git a/cpp/src/arrow/util/sort.h b/cpp/src/arrow/util/sort_internal.h
similarity index 100%
rename from cpp/src/arrow/util/sort.h
rename to cpp/src/arrow/util/sort_internal.h
diff --git a/cpp/src/arrow/util/spaced.h b/cpp/src/arrow/util/spaced_internal.h
similarity index 100%
rename from cpp/src/arrow/util/spaced.h
rename to cpp/src/arrow/util/spaced_internal.h
diff --git a/cpp/src/arrow/util/stl_util_test.cc b/cpp/src/arrow/util/stl_util_test.cc
index 3f16051f1df0..836469bc0655 100644
--- a/cpp/src/arrow/util/stl_util_test.cc
+++ b/cpp/src/arrow/util/stl_util_test.cc
@@ -22,7 +22,7 @@
 #include <gtest/gtest.h>
 
 #include "arrow/testing/gtest_util.h"
-#include "arrow/util/sort.h"
+#include "arrow/util/sort_internal.h"
 #include "arrow/util/string.h"
 #include "arrow/util/vector.h"
 
diff --git a/cpp/src/arrow/util/stopwatch.h b/cpp/src/arrow/util/stopwatch_internal.h
similarity index 100%
rename from cpp/src/arrow/util/stopwatch.h
rename to cpp/src/arrow/util/stopwatch_internal.h
diff --git a/cpp/src/arrow/util/tdigest.cc b/cpp/src/arrow/util/tdigest.cc
index ec92fabed8ed..36a83fb336d3 100644
--- a/cpp/src/arrow/util/tdigest.cc
+++ b/cpp/src/arrow/util/tdigest.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/tdigest.h"
+#include "arrow/util/tdigest_internal.h"
 
 #include <algorithm>
 #include <cmath>
diff --git a/cpp/src/arrow/util/tdigest_benchmark.cc b/cpp/src/arrow/util/tdigest_benchmark.cc
index d9cd632c3947..f0bf42341258 100644
--- a/cpp/src/arrow/util/tdigest_benchmark.cc
+++ b/cpp/src/arrow/util/tdigest_benchmark.cc
@@ -19,7 +19,7 @@
 
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/random.h"
-#include "arrow/util/tdigest.h"
+#include "arrow/util/tdigest_internal.h"
 
 namespace arrow {
 namespace util {
diff --git a/cpp/src/arrow/util/tdigest.h b/cpp/src/arrow/util/tdigest_internal.h
similarity index 100%
rename from cpp/src/arrow/util/tdigest.h
rename to cpp/src/arrow/util/tdigest_internal.h
diff --git a/cpp/src/arrow/util/tdigest_test.cc b/cpp/src/arrow/util/tdigest_test.cc
index 63395b676a61..04742ec46d8c 100644
--- a/cpp/src/arrow/util/tdigest_test.cc
+++ b/cpp/src/arrow/util/tdigest_test.cc
@@ -33,7 +33,7 @@
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/random.h"
 #include "arrow/testing/util.h"
-#include "arrow/util/tdigest.h"
+#include "arrow/util/tdigest_internal.h"
 
 namespace arrow {
 namespace internal {
diff --git a/cpp/src/arrow/util/trie.cc b/cpp/src/arrow/util/trie.cc
index 7862c86f38db..2b2a60154e6e 100644
--- a/cpp/src/arrow/util/trie.cc
+++ b/cpp/src/arrow/util/trie.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "arrow/util/trie.h"
+#include "arrow/util/trie_internal.h"
 
 #include <iostream>
 #include <utility>
diff --git a/cpp/src/arrow/util/trie_benchmark.cc b/cpp/src/arrow/util/trie_benchmark.cc
index b938f87d8d1c..27fbffac4d04 100644
--- a/cpp/src/arrow/util/trie_benchmark.cc
+++ b/cpp/src/arrow/util/trie_benchmark.cc
@@ -23,7 +23,7 @@
 
 #include "arrow/status.h"
 #include "arrow/testing/gtest_util.h"
-#include "arrow/util/trie.h"
+#include "arrow/util/trie_internal.h"
 
 namespace arrow {
 namespace internal {
diff --git a/cpp/src/arrow/util/trie.h b/cpp/src/arrow/util/trie_internal.h
similarity index 100%
rename from cpp/src/arrow/util/trie.h
rename to cpp/src/arrow/util/trie_internal.h
diff --git a/cpp/src/arrow/util/trie_test.cc b/cpp/src/arrow/util/trie_test.cc
index 9c6b7678a467..86d274178d69 100644
--- a/cpp/src/arrow/util/trie_test.cc
+++ b/cpp/src/arrow/util/trie_test.cc
@@ -26,7 +26,7 @@
 #include <gtest/gtest.h>
 
 #include "arrow/testing/gtest_util.h"
-#include "arrow/util/trie.h"
+#include "arrow/util/trie_internal.h"
 
 namespace arrow {
 namespace internal {
diff --git a/cpp/src/gandiva/gdv_function_stubs.cc b/cpp/src/gandiva/gdv_function_stubs.cc
index 7a47f7491a47..76bbdb902c0d 100644
--- a/cpp/src/gandiva/gdv_function_stubs.cc
+++ b/cpp/src/gandiva/gdv_function_stubs.cc
@@ -26,7 +26,7 @@
 
 #include "arrow/util/base64.h"
 #include "arrow/util/bit_util.h"
-#include "arrow/util/double_conversion.h"
+#include "arrow/util/double_conversion_internal.h"
 #include "arrow/util/value_parsing.h"
 
 #include "gandiva/encrypt_utils.h"
diff --git a/cpp/src/gandiva/gdv_string_function_stubs.cc b/cpp/src/gandiva/gdv_string_function_stubs.cc
index 17eefbe22e31..42af8fde6142 100644
--- a/cpp/src/gandiva/gdv_string_function_stubs.cc
+++ b/cpp/src/gandiva/gdv_string_function_stubs.cc
@@ -23,7 +23,7 @@
 #include <unordered_map>
 #include <vector>
 
-#include "arrow/util/double_conversion.h"
+#include "arrow/util/double_conversion_internal.h"
 #include "arrow/util/utf8_internal.h"
 #include "arrow/util/value_parsing.h"
 
diff --git a/cpp/src/parquet/decoder.cc b/cpp/src/parquet/decoder.cc
index 5382b0f6e7e0..ff21abcb1f26 100644
--- a/cpp/src/parquet/decoder.cc
+++ b/cpp/src/parquet/decoder.cc
@@ -43,7 +43,7 @@
 #include "arrow/util/int_util_overflow.h"
 #include "arrow/util/logging_internal.h"
 #include "arrow/util/rle_encoding_internal.h"
-#include "arrow/util/spaced.h"
+#include "arrow/util/spaced_internal.h"
 #include "arrow/util/ubsan.h"
 #include "arrow/visit_data_inline.h"
 
diff --git a/cpp/src/parquet/encoder.cc b/cpp/src/parquet/encoder.cc
index 2a759b8e0920..112b810a8f9d 100644
--- a/cpp/src/parquet/encoder.cc
+++ b/cpp/src/parquet/encoder.cc
@@ -40,7 +40,7 @@
 #include "arrow/util/int_util_overflow.h"
 #include "arrow/util/logging_internal.h"
 #include "arrow/util/rle_encoding_internal.h"
-#include "arrow/util/spaced.h"
+#include "arrow/util/spaced_internal.h"
 #include "arrow/util/ubsan.h"
 #include "arrow/visit_data_inline.h"
 
diff --git a/cpp/src/parquet/level_comparison.cc b/cpp/src/parquet/level_comparison.cc
index f3188e987d08..199e137ceab1 100644
--- a/cpp/src/parquet/level_comparison.cc
+++ b/cpp/src/parquet/level_comparison.cc
@@ -23,7 +23,7 @@
 
 #include <vector>
 
-#include "arrow/util/dispatch.h"
+#include "arrow/util/dispatch_internal.h"
 
 namespace parquet::internal {