[pull] master from tensorflow:master

configure.py

@@ -913,8 +913,16 @@ def retrieve_clang_version(clang_executable):
                            stderr=stderr)
 
   curr_version_split = curr_version.lower().split('clang version ')
-  if len(curr_version_split) > 1:
-    curr_version = curr_version_split[1].split()[0].split('git')
+  if len(curr_version_split) <= 1:
+    sys.stdout.write('WARNING: current clang installation version unknown.\n')
+    return None
+
+  tokens = curr_version_split[1].split()
+  if not tokens:
+    sys.stdout.write('WARNING: current clang installation version unknown.\n')
+    return None
+
+  curr_version = tokens[0].split('git')
 
   if len(curr_version) > 1:
     print('WARNING: current clang installation is not a release version.\n')
@@ -937,7 +945,13 @@ def retrieve_clang_version(clang_executable):
 # offset of in the current version of ubp. See
 # https://github.com/protocolbuffers/upb/blob/9effcbcb27f0a665f9f345030188c0b291e32482/upb/upb.c#L183.
 def disable_clang_offsetof_extension(clang_version):
-  if int(clang_version.split('.')[0]) in (16, 17):
+  if not clang_version:
+    return
+  try:
+    clang_major_version = int(clang_version.split('.')[0])
+  except ValueError:
+    return
+  if clang_major_version in (16, 17):
     write_to_bazelrc('build --copt=-Wno-gnu-offsetof-extensions')
 
 

tensorflow/compiler/mlir/lite/flatbuffer_export.cc

@@ -4518,6 +4518,7 @@ absl::Status Translator::AppendBufferData() {
   for (const auto& [index, buffer] : const_buffer_storage_.buffers()) {
     uint64_t hash = buffer->hash();
     if (hashcode_to_pos.find(hash) == hashcode_to_pos.end()) {
+      export_stream_.get().write_zeros(kFbAlignment - offset() % kFbAlignment);
       int64_t size = 0;
       int64_t buffer_offset = offset();
       auto status = buffer->ApplyData([this, &size](absl::string_view data) {

tensorflow/compiler/mlir/tensorflow/transforms/cluster_ops_by_policy.h

@@ -200,7 +200,7 @@ class ClusteringPolicySet {
  private:
   template <typename T, typename... Args>
   void AddImpl(Args&&... args) {
-    static_assert(std::is_base_of<ClusteringPolicy, T>::value,
+    static_assert(std::is_base_of_v<ClusteringPolicy, T>,
                   "T must implement ClusteringPolicy");
     policies_.emplace_back(std::make_unique<T>(std::forward<Args>(args)...));
   }

tensorflow/compiler/mlir/tensorflow/transforms/fused_kernel_matcher.cc

@@ -199,7 +199,7 @@ class FuseContractionWithBiasAdd : public OpRewritePattern<SrcOpT> {
     attrs.push_back(
         NamedAttribute(StringAttr::get(context, "epsilon"), epsilon));
 
-    if (std::is_same<FusedOpT, _FusedConv2DOp>::value) {
+    if (std::is_same_v<FusedOpT, _FusedConv2DOp>) {
       // Here TArgs types do not include types of the first two parameters,
       // i.e. the convolution input and the filter. TArgs are parameters for
       // the extras like the bias etc.

tensorflow/core/data/service/dispatcher_impl.cc

@@ -205,12 +205,16 @@ absl::Status ValidateDatasetId(const std::string& dataset_id) {
         absl::StrCat("Invalid dataset ID: ", dataset_id,
                      ". Dataset IDs must not contain '/'."));
   }
+  if (absl::StrContains(dataset_id, '\\')) {
+    return absl::InvalidArgumentError(
+        absl::StrCat("Invalid dataset ID: ", dataset_id,
+                     ". Dataset IDs must not contain '\\'."));
+  }
 #if defined(_WIN32)
-  if (absl::StrContains(dataset_id, '\\') ||
-      absl::StrContains(dataset_id, ':')) {
+  if (absl::StrContains(dataset_id, ':')) {
     return absl::InvalidArgumentError(
         absl::StrCat("Invalid dataset ID: ", dataset_id,
-                     ". Dataset IDs must not contain '\\' or ':'."));
+                     ". Dataset IDs must not contain ':'."));
   }
 #endif
   return absl::OkStatus();

tensorflow/core/data/service/grpc_dispatcher_impl_test.cc

@@ -175,6 +175,24 @@ TEST_F(GrpcDispatcherImplTest, GetSplitInvalidProviderIndex) {
   }
 }
 
+TEST_F(GrpcDispatcherImplTest, GetOrRegisterDatasetInvalidDatasetId) {
+  const std::vector<std::string> invalid_ids = {
+      "..\\..\\etc\\passwd", "a\\b", "a/b", ".", "..",
+  };
+
+  for (const auto& id : invalid_ids) {
+    ClientContext ctx;
+    GetOrRegisterDatasetRequest req;
+    *req.mutable_dataset()->mutable_graph() = testing::RangeDataset(10).graph();
+    req.set_dataset_id(id);
+    GetOrRegisterDatasetResponse resp;
+    ::grpc::Status status =
+        dispatcher_client_stub_->GetOrRegisterDataset(&ctx, req, &resp);
+    EXPECT_EQ(status.error_code(), ::grpc::StatusCode::INVALID_ARGUMENT)
+        << "Dataset ID '" << id << "' should be rejected.";
+  }
+}
+
 }  // namespace
 }  // namespace data
 }  // namespace tensorflow

tensorflow/core/grappler/optimizers/constant_folding.cc

@@ -2042,17 +2042,6 @@ void ConstantFolding::ReplaceBinaryOperationWithBroadcastTo(
   graph_modified_ = true;
 }
 
-void ConstantFolding::ReplaceDivisionOfOnesByReciprocal(NodeDef* node,
-                                                        GraphDef* graph) {
-  node->set_op("Reciprocal");
-  node->mutable_input()->SwapElements(0, 1);
-  const std::string ctrl_dep =
-      AddControlDependency(node->input(1), graph, node_map_.get());
-  node_map_->UpdateInput(node->name(), node->input(1), ctrl_dep);
-  node->set_input(1, ctrl_dep);
-  graph_modified_ = true;
-}
-
 void ConstantFolding::ReplaceSubtractionFromZeroByNegation(NodeDef* node,
                                                            GraphDef* graph) {
   node->set_op("Neg");
@@ -3047,15 +3036,10 @@ absl::Status ConstantFolding::SimplifyArithmeticOperations(
       return absl::OkStatus();
     }
 
-    // Replace 1 / y with Reciprocal op.
-    if (y_matches_output_shape && is_any_div && x_is_one) {
-      TF_RETURN_IF_ERROR(CheckAttrExists(*node, "T"));
-      DataType type = node->attr().at("T").type();
-      if (DataTypeIsFloating(type) || DataTypeIsComplex(type)) {
-        ReplaceDivisionOfOnesByReciprocal(node, optimized_graph);
-        return absl::OkStatus();
-      }
-    }
+    // Note: 1 / y is intentionally not rewritten to Reciprocal(y). The CPU
+    // Reciprocal kernel uses Eigen's fast-math reciprocal for float, which
+    // is not exactly IEEE division, so the rewrite silently changed results
+    // between eager and graph execution on x86 (see issue #102771).
 
     const bool y_is_zero = IsZeros(*y);
     const bool y_is_one = y_is_zero ? false : IsOnes(*y);

tensorflow/core/grappler/optimizers/constant_folding.h

@@ -131,7 +131,6 @@ class ConstantFolding : public GraphOptimizer {
                                                   NodeDef* node,
                                                   GraphDef* graph);
 
-  void ReplaceDivisionOfOnesByReciprocal(NodeDef* node, GraphDef* graph);
   absl::Status FoldGraph(
       const GraphProperties& properties, GraphDef* output,
       absl::flat_hash_set<std::string>* nodes_to_not_simplify);

tensorflow/core/grappler/optimizers/constant_folding_test.cc

@@ -888,9 +888,11 @@ TEST_F(ConstantFoldingTest, NeutralElement) {
         EXPECT_EQ("x", node.input(0));
         EXPECT_EQ(ctrl_ones_name, node.input(1));
       } else if (name == "div2") {
-        EXPECT_EQ("Reciprocal", node.op());
-        EXPECT_EQ("y", node.input(0));
-        EXPECT_EQ(ctrl_ones_name, node.input(1));
+        // ones / y is not rewritten to Reciprocal(y): the CPU Reciprocal
+        // kernel is not exactly IEEE division for float (see issue #102771).
+        EXPECT_EQ("Div", node.op());
+        EXPECT_EQ(ones_name, node.input(0));
+        EXPECT_EQ("y", node.input(1));
       } else if (name == "floordiv") {
         EXPECT_EQ("FloorDiv", node.op());
         EXPECT_EQ("x", node.input(0));

tensorflow/core/util/tensor_slice_reader_cache.cc

@@ -52,8 +52,6 @@ TensorSliceReaderCache::~TensorSliceReaderCache() {
 const TensorSliceReader* TensorSliceReaderCache::GetReader(
     const std::string& filepattern,
     TensorSliceReader::OpenTableFunction open_function, int preferred_shard) {
-  mutex_lock l(mu_);
-
 #if defined(__GXX_RTTI) || defined(_CPPRTTI)
   // Get the function pointer from the open_function value.
   TensorSliceReaderCache::OpenFuncType* func_ptr =
@@ -72,22 +70,43 @@ const TensorSliceReader* TensorSliceReaderCache::GetReader(
     return nullptr;
   }
 
-  // Wait if another thread is already trying to open the same files.
-  while (still_opening_.find(filepattern) != still_opening_.end()) {
-    cv_.wait(l);
+  TensorSliceReader* reader = nullptr;
+
+  // scope block for lock
+  {
+    mutex_lock l(mu_);
+
+    // Wait if another thread is already trying to open the same files.
+    while (still_opening_.find(filepattern) != still_opening_.end()) {
+      cv_.wait(l);
+    }
+
+    auto it = readers_.find(filepattern);
+    if (it != readers_.end()) {
+      auto cached_val = it->second;
+      if (cached_val.first == *func_ptr) {
+        reader = cached_val.second;
+        VLOG(1) << "Using cached TensorSliceReader for " << filepattern << ": "
+                << reader;
+      } else {
+        LOG(WARNING) << "Caching disabled because the checkpoint file "
+                     << "is being opened with two different open functions: "
+                     << filepattern;
+      }
+      return reader;
+    }
+
+    still_opening_.insert(filepattern);
   }
 
-  TensorSliceReader* reader = nullptr;
-  if (readers_.find(filepattern) == readers_.end()) {
+  // no lock for expensive constructing TensorSliceReader
+  TensorSliceReader* tmp_reader(
+      new TensorSliceReader(filepattern, open_function, preferred_shard));
+
+  // scope block for lock
+  {
     VLOG(1) << "Creating new TensorSliceReader for " << filepattern;
-    still_opening_.insert(filepattern);
-    // Release the lock temporary as constructing TensorSliceReader is
-    // expensive.
-    mu_.unlock();
-    TensorSliceReader* tmp_reader(
-        new TensorSliceReader(filepattern, open_function, preferred_shard));
-    // Acquire the lock again.
-    mu_.lock();
+    mutex_lock l(mu_);
     if (tmp_reader->status().ok()) {
       reader = tmp_reader;
       readers_[filepattern] = std::make_pair(*func_ptr, reader);
@@ -96,20 +115,9 @@ const TensorSliceReader* TensorSliceReaderCache::GetReader(
     }
     CHECK_EQ(size_t{1}, still_opening_.erase(filepattern));
     VLOG(1) << "Cached TensorSliceReader for " << filepattern << ": " << reader;
-  } else {
-    auto cached_val = readers_[filepattern];
-    if (cached_val.first == *func_ptr) {
-      reader = cached_val.second;
-      VLOG(1) << "Using cached TensorSliceReader for " << filepattern << ": "
-              << reader;
-    } else {
-      LOG(WARNING) << "Caching disabled because the checkpoint file "
-                   << "is being opened with two different open functions: "
-                   << filepattern;
-    }
-  }
 
-  cv_.notify_all();
+    cv_.notify_all();
+  }
   return reader;
 }
 

tensorflow/lite/python/lite_v2_test.py

@@ -5592,6 +5592,86 @@ def testCOncreteFunctionFloat(self):
     actual_value = self._evaluateTFLiteModel(tflite_model, [input_data])
     self.assertEqual(expected_value.numpy(), actual_value)
 
+  @test_util.run_v2_only
+  def testUseBufferOffsetAlignment(self):
+    """Test that all external constant buffers are aligned to 16 bytes."""
+
+    class MultiConstantModel(tf.Module):
+
+      def __init__(self):
+        super().__init__()
+        # Constants with non-multiple-of-16 byte sizes to test padding alignment
+        self.w1 = tf.Variable(
+            tf.ones([3, 1], dtype=tf.float32), name='w1'
+        )  # 3 * 4 = 12 bytes
+        self.w2 = tf.Variable(
+            tf.ones([5, 1], dtype=tf.float32), name='w2'
+        )  # 5 * 4 = 20 bytes
+        self.w3 = tf.Variable(
+            tf.ones([1, 1], dtype=tf.float32), name='w3'
+        )  # 1 * 4 = 4 bytes
+        self.w4 = tf.Variable(
+            tf.ones([7, 1], dtype=tf.float32), name='w4'
+        )  # 7 * 4 = 28 bytes
+        self.w5 = tf.Variable(
+            tf.ones([6, 1], dtype=tf.float32), name='w5'
+        )  # 6 * 4 = 24 bytes
+
+      @tf.function
+      def __call__(self, x1, x2, x3, x4, x5):
+        return (
+            tf.matmul(x1, self.w1)
+            + tf.matmul(x2, self.w2)
+            + tf.matmul(x3, self.w3)
+            + tf.matmul(x4, self.w4)
+            + tf.matmul(x5, self.w5)
+        )
+
+    root = MultiConstantModel()
+    inputs = [
+        tf.constant([[1.0, 2.0, 3.0]], dtype=tf.float32),
+        tf.constant([[1.0, 2.0, 3.0, 4.0, 5.0]], dtype=tf.float32),
+        tf.constant([[1.0]], dtype=tf.float32),
+        tf.constant([[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0]], dtype=tf.float32),
+        tf.constant([[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]], dtype=tf.float32),
+    ]
+    concrete_func = root.__call__.get_concrete_function(*inputs)
+
+    converter = lite.TFLiteConverterV2.from_concrete_functions(
+        [concrete_func], root
+    )
+    converter._experimental_use_buffer_offset = True
+    tflite_model = converter.convert()
+
+    # Parse flatbuffer model and check all buffer offsets
+    model_obj = schema_fb.Model.GetRootAsModel(tflite_model, 0)
+    external_buffer_count = 0
+    for i in range(model_obj.BuffersLength()):
+      buf = model_obj.Buffers(i)
+      if buf.Offset() > 1:
+        external_buffer_count += 1
+        self.assertEqual(
+            buf.Offset() % 16,
+            0,
+            f'Buffer {i} offset {buf.Offset()} is not 16-byte aligned (offset %'
+            f' 16 = {buf.Offset() % 16})',
+        )
+    self.assertGreaterEqual(external_buffer_count, 5)
+
+    # Evaluate converted model
+    expected_value = root(*inputs)
+    interp = interpreter.Interpreter(model_content=tflite_model)
+    runner = interp.get_signature_runner()
+    output = runner(
+        x1=inputs[0],
+        x2=inputs[1],
+        x3=inputs[2],
+        x4=inputs[3],
+        x5=inputs[4],
+    )
+    actual_value = list(output.values())[0]
+    self.assertEqual(expected_value.numpy(), actual_value)
+
   @test_util.run_v2_only
   def testConcreteFunctionStringInput(self):
     class Model(tf.Module):

tensorflow/python/_pywrap_tensorflow.def

@@ -256,6 +256,7 @@ EXPORTS
  ?ComputeGradient@Tape@gradients@tensorflow@@QEAA?AVStatus@lts_20260526@absl@@PEAVAbstractContext@3@V?$Span@QEAVAbstractTensorHandle@tensorflow@@@56@11V?$Span@PEAVAbstractTensorHandle@tensorflow@@@56@@Z
  ?Convert@PythonTensorConverter@tensorflow@@QEBA?AV?$unique_ptr@U_object@@UPyDecrefDeleter@detail@tensorflow@@@std@@PEAU_object@@AEAW4DataType@2@PEA_N@Z
  ?Convert@tflite@@YAPEAU_object@@PEAU2@00_N0PEBVPyFunctionLibrary@quantization@tensorflow@@@Z
+ ?ConvertMlirBytecode@tflite@@YAPEAU_object@@PEAU2@00@Z
  ?ConvertPyObjectToAttributeType@tensorflow@@YA?AV?$unique_ptr@U_object@@UPyDecrefDeleter@detail@tensorflow@@@std@@PEAU_object@@W4AttributeType@1@@Z
  ?ConvertPythonAPIParameters@tensorflow@@YA_NAEBVPythonAPIInfo@1@AEBVPythonTensorConverter@1@V?$Span@PEAU_object@@@lts_20260526@absl@@PEAUInferredAttributes@21@@Z
  ?ConvertToEagerTensor@tensorflow@@YAPEAUTFE_TensorHandle@@PEAUTFE_Context@@PEAU_object@@W4DataType@1@PEBD@Z

tensorflow/python/data/kernel_tests/BUILD

@@ -1422,6 +1422,7 @@ py_test(
         ":checkpoint_test_base",
         ":test_base",
         "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/ops:debug_mode",
         "//tensorflow/python/eager:def_function",
         "//tensorflow/python/framework:combinations",
         "//tensorflow/python/ops:variables",

tensorflow/python/data/kernel_tests/io_test.py

@@ -21,9 +21,11 @@
 
 from absl.testing import parameterized
 import numpy as np
+
 from tensorflow.python.data.kernel_tests import checkpoint_test_base
 from tensorflow.python.data.kernel_tests import test_base
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.ops import debug_mode
 from tensorflow.python.eager import def_function
 from tensorflow.python.framework import combinations
 from tensorflow.python.ops import variables
@@ -70,6 +72,15 @@ def testCardinality(self):
     dataset2 = dataset_ops.Dataset.load(self._test_dir, dataset.element_spec)
     self.assertEqual(self.evaluate(dataset2.cardinality()), 42)
 
+  @combinations.generate(test_base.eager_only_combinations())
+  def testSaveInDebugModeWithoutShardFunction(self):
+    debug_mode.toggle_debug_mode(True)
+    self.addCleanup(debug_mode.toggle_debug_mode, False)
+    dataset = dataset_ops.Dataset.range(42)
+    self.evaluate(dataset.save(self._test_dir))
+    dataset2 = dataset_ops.Dataset.load(self._test_dir, dataset.element_spec)
+    self.assertDatasetProduces(dataset2, range(42))
+
   @combinations.generate(test_base.default_test_combinations())
   def testCustomShardFunction(self):
     dataset = dataset_ops.Dataset.range(42)

tensorflow/python/data/ops/save_op.py

@@ -97,7 +97,7 @@ def set_save_dataset_attributes(dataset, shard_func, path):
   """Sets parameters for SaveDatasetOp and SaveDatasetV2Op."""
   if shard_func is None:
     use_shard_func = False
-    shard_func = lambda *x: None  # a dummy function that will not be used
+    shard_func = lambda *x: 0  # a dummy function that will not be used
   else:
     use_shard_func = True
   wrapped_func = structured_function.StructuredFunctionWrapper(