variable sized root ranges

Author: yoid2000

syndiffix/anonymizer.py

@@ -39,6 +39,13 @@ def _random_uniform(interval: FlatteningInterval, seed: Hash) -> int:
     return int(seed) % (interval.upper - interval.lower + 1) + interval.lower
 
 
+def _random_uniform_float(lower: float, upper: float, seed: Hash) -> float:
+    # Convert hash to a float between 0 and 1
+    u = (int(seed) & 0x7FFFFFFFFFFFFFFF) / 0x7FFFFFFFFFFFFFFF
+    # Scale to the desired range
+    return lower + u * (upper - lower)
+
+
 def _random_normal(sd: float, seed: Hash) -> float:
     u1 = (int(seed) & 0x7FFFFFFF) / 0x7FFFFFFF
     u1 = max(u1, sys.float_info.epsilon)
@@ -293,3 +300,16 @@ def noisy_row_limit(salt: bytes, seed: Hash, row_count: int, row_fraction: int)
     noise = _random_uniform(FlatteningInterval(-noise_range, noise_range), noise_seed)
 
     return real_row_limit + noise
+
+
+def generate_root_buffers(anonymization_context: AnonymizationContext) -> tuple[float, float]:
+    root_buffers = anonymization_context.anonymization_params.root_buffers
+    salt = anonymization_context.anonymization_params.salt
+
+    lower_seed = _crypto_hash_salted_seed(salt, _mix_seed("lower_buffer", anonymization_context.bucket_seed))
+    upper_seed = _crypto_hash_salted_seed(salt, _mix_seed("upper_buffer", anonymization_context.bucket_seed))
+
+    lower_buffer = _random_uniform_float(root_buffers.lower_low, root_buffers.lower_high, lower_seed)
+    upper_buffer = _random_uniform_float(root_buffers.upper_low, root_buffers.upper_high, upper_seed)
+
+    return lower_buffer, upper_buffer

syndiffix/common.py

@@ -33,6 +33,14 @@ class Column:
 Columns = tuple[Column, ...]
 
 
+@dataclass(frozen=True)
+class RootBuffer:
+    upper_high: float = 0.95
+    upper_low: float = 0.85
+    lower_high: float = 0.15
+    lower_low: float = 0.05
+
+
 @dataclass(frozen=True)
 class FlatteningInterval:
     lower: int = 2
@@ -63,6 +71,7 @@ class AnonymizationParams:
     low_count_params: SuppressionParams = SuppressionParams()
     outlier_count: FlatteningInterval = FlatteningInterval()
     top_count: FlatteningInterval = FlatteningInterval()
+    root_buffers: RootBuffer = RootBuffer()
     layer_noise_sd: float = 1.0
 
 

syndiffix/microdata.py

@@ -17,8 +17,16 @@
 )
 from sklearn.preprocessing import MinMaxScaler
 
+from .anonymizer import generate_root_buffers, hash_strings
 from .bucket import Buckets
-from .common import ColumnId, ColumnType, Value, check_column_names_or_ids
+from .common import (
+    AnonymizationContext,
+    AnonymizationParams,
+    ColumnId,
+    ColumnType,
+    Value,
+    check_column_names_or_ids,
+)
 from .interval import Interval, Intervals
 from .tree import Branch, Leaf, Node
 
@@ -33,10 +41,15 @@
 
 
 class DataConvertor(ABC):
-    def __init__(self) -> None:
+    def __init__(self, column: str, anonymization_params: AnonymizationParams) -> None:
         self.scaler: Optional[MinMaxScaler] = None
         self.value_safe_flag: bool = False
 
+        base_seed = hash_strings(iter([str(column)]))
+        self.lower_buffer, self.upper_buffer = generate_root_buffers(
+            AnonymizationContext(base_seed, anonymization_params)
+        )
+
     @abstractmethod
     def column_type(self) -> ColumnType:
         pass
@@ -64,8 +77,8 @@ def denormalize_safe_values(self) -> None:
 
 
 class BooleanConvertor(DataConvertor):
-    def __init__(self) -> None:
-        super().__init__()
+    def __init__(self, column: str, anonymization_params: AnonymizationParams) -> None:
+        super().__init__(column, anonymization_params)
 
     def column_type(self) -> ColumnType:
         return ColumnType.BOOLEAN
@@ -84,13 +97,12 @@ def create_value_safe_set(self, values: pd.Series) -> None:
 
 
 class RealConvertor(DataConvertor):
-    def __init__(self, values: Iterable[Value]) -> None:
-        super().__init__()
-        # Fit up to 0.9999 so that the max bucket range is [0-1)
-        self.scaler = MinMaxScaler(feature_range=(0.0, 0.9999))  # type: ignore
+    def __init__(self, column: str, anonymization_params: AnonymizationParams, values: Iterable[Value]) -> None:
+        super().__init__(column, anonymization_params)
+        self.scaler = MinMaxScaler(feature_range=(self.lower_buffer, self.upper_buffer))  # type: ignore
         # This value-neutral fitting is only for passing unit tests, gets overridden
         # later by fit_transform().
-        self.scaler.fit(np.array([[0.0], [0.9999]]))
+        self.scaler.fit(np.array([[self.lower_buffer], [self.upper_buffer]]))
         self.final_round_precision = _get_round_precision(cast(Iterable[float], values))
 
     def column_type(self) -> ColumnType:
@@ -116,13 +128,12 @@ def create_value_safe_set(self, values: pd.Series) -> None:
 
 
 class IntegerConvertor(DataConvertor):
-    def __init__(self) -> None:
-        super().__init__()
-        # Fit up to 0.9999 so that the max bucket range is [0-1)
-        self.scaler = MinMaxScaler(feature_range=(0.0, 0.9999))  # type: ignore
+    def __init__(self, column: str, anonymization_params: AnonymizationParams) -> None:
+        super().__init__(column, anonymization_params)
+        self.scaler = MinMaxScaler(feature_range=(self.lower_buffer, self.upper_buffer))  # type: ignore
         # This value-neutral fitting is only for passing unit tests, gets overridden
         # later by fit_transform().
-        self.scaler.fit(np.array([[0.0], [0.9999]]))
+        self.scaler.fit(np.array([[self.lower_buffer], [self.upper_buffer]]))
 
     def column_type(self) -> ColumnType:
         return ColumnType.INTEGER
@@ -146,13 +157,12 @@ def create_value_safe_set(self, values: pd.Series) -> None:
 
 
 class TimestampConvertor(DataConvertor):
-    def __init__(self) -> None:
-        super().__init__()
-        # Fit up to 0.9999 so that the max bucket range is [0-1)
-        self.scaler = MinMaxScaler(feature_range=(0.0, 0.9999))  # type: ignore
+    def __init__(self, column: str, anonymization_params: AnonymizationParams) -> None:
+        super().__init__(column, anonymization_params)
+        self.scaler = MinMaxScaler(feature_range=(self.lower_buffer, self.upper_buffer))  # type: ignore
         # This value-neutral fitting is only for passing unit tests, gets overridden
         # later by fit_transform().
-        self.scaler.fit(np.array([[0.0], [0.9999]]))
+        self.scaler.fit(np.array([[self.lower_buffer], [self.upper_buffer]]))
 
     def column_type(self) -> ColumnType:
         return ColumnType.TIMESTAMP
@@ -177,8 +187,8 @@ def create_value_safe_set(self, values: pd.Series) -> None:
 
 
 class StringConvertor(DataConvertor):
-    def __init__(self, values: Iterable[Value]) -> None:
-        super().__init__()
+    def __init__(self, column: str, anonymization_params: AnonymizationParams, values: Iterable[Value]) -> None:
+        super().__init__(column, anonymization_params)
         unique_values = set()
         for v in values:
             if not pd.isna(v):
@@ -196,7 +206,6 @@ def __init__(self, values: Iterable[Value]) -> None:
 
         # Note that self.safe_values is only used if self.value_safe_flag is False
         self.safe_values: Set[float] = set()
-        # Fit up to 0.9999 so that the max bucket range is [0-1)
         self.scaler = MinMaxScaler(feature_range=(0.0, 0.9999))  # type: ignore
         # This value-neutral fitting is only for passing unit tests, gets overridden
         # later by fit_transform().
@@ -231,7 +240,7 @@ def _map_interval(self, interval: Interval, rng: Random) -> MicrodataValue:
         min_value = int(interval.min)
         # max_value is inclusive
         max_value = min(int(interval.max) - 1, len(self.value_map) - 1)
-        # The latter term in the above line can 0 (not sure why TODO: check)
+        # The latter term in the above line can be 0 (not sure why TODO: check)
         max_value = max(min_value, max_value)
         value = rng.randint(min_value, max_value)
         if self.value_safe_flag is True or value in self.safe_values:
@@ -343,19 +352,19 @@ def _microdata_row_generator(
         yield [_generate(i, c, nm, rng) for i, c, nm in zip(intervals, convertors, null_mappings)]
 
 
-def get_convertor(df: pd.DataFrame, column: str) -> DataConvertor:
+def get_convertor(df: pd.DataFrame, column: str, anonymization_params: AnonymizationParams) -> DataConvertor:
     dtype = df.dtypes[column]
     if is_integer_dtype(dtype):
-        return IntegerConvertor()
+        return IntegerConvertor(column, anonymization_params)
     elif is_float_dtype(dtype):
-        return RealConvertor(df[column])
+        return RealConvertor(column, anonymization_params, df[column])
     elif is_bool_dtype(dtype):
-        return BooleanConvertor()
+        return BooleanConvertor(column, anonymization_params)
     elif is_datetime64_dtype(dtype):
-        return TimestampConvertor()
+        return TimestampConvertor(column, anonymization_params)
     elif is_string_dtype(dtype):
         # Note above is `True` for `object` dtype, but `StringConvertor` will assert values are `str`.
-        return StringConvertor(df[column])
+        return StringConvertor(column, anonymization_params, df[column])
     else:
         raise TypeError(f"Dtype {dtype} is not supported.")
 

syndiffix/synthesizer.py

@@ -98,9 +98,10 @@ def __init__(
             )
             counters_factory = GenericPidCountersFactory(len(pids.columns), max_low_count)
 
+        self.columns = tuple(raw_data.columns)
         self.raw_dtypes = raw_data.dtypes
 
-        self.column_convertors = [get_convertor(raw_data, column) for column in raw_data.columns]
+        self.column_convertors = [get_convertor(raw_data, column, anonymization_params) for column in raw_data.columns]
         for col_id, convertor in enumerate(self.column_convertors):
             convertor.set_value_safe_flag(self.value_safe_columns_array[col_id])
         self.column_is_integral = [self._is_integral(convertor.column_type()) for convertor in self.column_convertors]

tests/clustering/test_measures.py

@@ -20,12 +20,13 @@ def test_measure_all() -> None:
         np.array(
             [
                 [1.00, 0.2, 0.15, 0.02, 0.05],
-                [0.2, 1.00, 0.18, 0.02, 0.06],
-                [0.15, 0.18, 1.00, 0.04, 0.07],
+                [0.2, 1.00, 0.14, 0.02, 0.04],
+                [0.15, 0.14, 1.00, 0.04, 0.08],
                 [0.02, 0.02, 0.04, 1.00, 0.01],
-                [0.05, 0.06, 0.07, 0.01, 1.00],
+                [0.05, 0.04, 0.08, 0.01, 1.00],
             ]
         ),
     )
 
-    assert np.array_equal(np.round(measures.entropy_1dim, 3), np.array([9.214, 9.207, 5.160, 0.118, 1.350]))
+    print(np.round(measures.entropy_1dim, 3))
+    assert np.array_equal(np.round(measures.entropy_1dim, 3), np.array([9.218, 9.212, 5.164, 0.118, 1.350]))

tests/conftest.py

@@ -9,9 +9,12 @@
 
 SALT = bytes([])
 NOISELESS_SUPPRESSION = SuppressionParams(layer_sd=0.0)
+NOISELESS_ROOT_BUFFERS = RootBuffer(upper_high=0.9, upper_low=0.9, lower_high=0.1, lower_low=0.1)
+# NOISELESS_ROOT_BUFFERS = RootBuffer(upper_high=1.0, upper_low=1.0, lower_high=0.0, lower_low=0.0)
 
 NOISELESS_PARAMS = AnonymizationParams(
     low_count_params=NOISELESS_SUPPRESSION,
+    root_buffers=NOISELESS_ROOT_BUFFERS,
     layer_noise_sd=0.0,
     outlier_count=FlatteningInterval(upper=FlatteningInterval().lower),
     top_count=FlatteningInterval(upper=FlatteningInterval().lower),
@@ -41,7 +44,9 @@ def _load_csv(path: str, columns: list[str] | None) -> pd.DataFrame:
     df = pd.read_csv(path, keep_default_na=False, na_values=[""], low_memory=False)
     if columns is not None:
         df = df[columns]
-    return apply_convertors([get_convertor(df, column) for column in df.columns], df)
+    # Create default anonymization params for loading CSV data
+    params = NOISELESS_PARAMS
+    return apply_convertors([get_convertor(df, column, params) for column in df.columns], df)
 
 
 def load_forest(