[executorch][native] Let quantized weights feed ops directly (pytorch#23336)

This PR was created by the merge bot to help merge the original PR into
the main branch.
ghstack PR number: pytorch#23231 by
@digantdesai
^ Please use this as the source of truth for the PR details, comments,
and reviews
ghstack PR base:
https://github.com/pytorch/executorch/tree/gh/digantdesai/111/base
ghstack PR head:
https://github.com/pytorch/executorch/tree/gh/digantdesai/111/head
Merge bot PR base:
https://github.com/pytorch/executorch/tree/gh/digantdesai/110/orig
Merge bot PR head:
https://github.com/pytorch/executorch/tree/gh/digantdesai/111/orig
Differential Revision:
[D121704320](https://our.internmc.facebook.com/intern/diff/D121704320/)
@diff-train-skip-merge

Co-authored-by: Digant Desai <digantdesai@meta.com>

Author: pytorchbot

backends/native/runtime/graph/TensorMeta.h

@@ -17,6 +17,8 @@
 
 namespace ptn {
 
+// See native_graph.fbs AffineGroup. As a direct op operand, a tensor carrying
+// this scheme denotes its decoded value, with dtype scale_dtype.
 struct AffineGroupQuant {
   std::string scale_data_key;
   ScalarType scale_dtype = ScalarType::Float;

backends/native/runtime/graph/Value.cpp

@@ -10,6 +10,14 @@
 
 namespace ptn {
 
+TensorMeta& Value::tensor_meta() {
+  TensorMeta* m = std::get_if<TensorMeta>(&value_);
+  if (m == nullptr) {
+    throw std::runtime_error("Value is not a tensor");
+  }
+  return *m;
+}
+
 const TensorMeta& Value::tensor_meta() const {
   const TensorMeta* m = std::get_if<TensorMeta>(&value_);
   if (m == nullptr) {

backends/native/runtime/graph/Value.h

@@ -111,6 +111,7 @@ class Value {
   }
 
   // Typed payload accessors: throw std::runtime_error unless the kind matches.
+  TensorMeta& tensor_meta();
   const TensorMeta& tensor_meta() const;
   const Scalar& scalar() const;
   const std::vector<ValueId>& content_ids() const;

backends/native/serialization/native_graph.fbs

@@ -116,12 +116,17 @@ enum OutputValueKind : byte {
 // axis). Scales (and, if asymmetric, zero-points) are shipped out-of-line via the
 // external constant file, keyed by data_key (empty zero_point_data_key means symmetric).
 // Their shapes derive from the weight shape and group_size, but their dtypes do not,
-// so they are recorded here. The dequant result dtype is the op output value's
-// TensorMeta.dtype, not part of this scheme. The qdata itself (the enclosing
-// TensorMeta) uses dtype=BYTE with logical sizes; its packed byte length follows from
-// the logical shape and bitwidth, not prod(sizes). The scale and zero-point keys must
+// so they are recorded here. The qdata itself (the enclosing TensorMeta) uses
+// dtype=BYTE with logical sizes. Sub-byte packing is flat over the elements in
+// memory order with no per-row padding, so the packed byte length is
+// ceil(numel * bitwidth / 8), not prod(sizes). The scale and zero-point keys must
 // be read-only stored constants of every method that uses the quantized tensor,
 // whether that tensor is itself a constant or a graph value such as a user input.
+//
+// A quantized tensor may feed an explicit dequant op, whose output TensorMeta.dtype
+// gives the result dtype, or it may be a direct op operand. A direct operand denotes
+// its decoded value with dtype scale_dtype; an engine must lower the consuming op,
+// materialize the decode, or reject the program.
 table AffineGroup {
     scale_data_key: string (required);
     scale_dtype: ScalarType;        // FLOAT, HALF, or BFLOAT16; not derivable