Better handling of safe values

Author: yoid2000

docs/development.md

@@ -10,6 +10,9 @@ Activate `poetry` environment: `poetry shell`. (can skip, then prepend `poetry r
 - Test: `pytest .`
 - Check: `flake8 . && mypy . && black --check . && isort . --check`
 
+ To filter out acceptable lines:
+     `poetry run flake8 . | findstr /v "E501" | findstr /v "F405" | findstr /v "F403"`
+
 ### Creating a new release
 
 First, update the project's version by editing the `[tool.poetry]` section from the `pyproject.toml` file.

pyproject.toml

@@ -17,7 +17,7 @@ profile = "black"
 
 [tool.poetry]
 name = "syndiffix"
-version = "1.0.7"
+version = "1.0.8"
 description = "Python implementation of the SynDiffix synthetic data generation mechanism."
 authors = ["Open Diffix <hello@open-diffix.org>"]
 readme = "README.md"

syndiffix/anonymizer.py

@@ -152,7 +152,7 @@ def _flatten_contributions(pid_contributions: PidContributions, context: Anonymi
     top_count = _random_uniform(top_interval, _mix_seed("top", flat_seed))
 
     top_group_sum = sum(
-        contribution for _, contribution in sorted_value_counts[outlier_count : (outlier_count + top_count)]
+        contribution for _, contribution in sorted_value_counts[outlier_count: (outlier_count + top_count)]
     )
     top_group_average = top_group_sum / top_count
 

syndiffix/bucket.py

@@ -202,7 +202,7 @@ def _refine_buckets(node: Node, harvested_nodes: BucketsCache, count: int, unsaf
 
 def _harvest_leaf(leaf: Leaf, harvested_nodes: BucketsCache, unsafe_rng: Random) -> Buckets:
     low_threshold = leaf.context.anonymization_context.anonymization_params.low_count_params.low_threshold
-    if leaf.is_over_threshold(low_threshold):
+    if leaf.is_over_threshold(low_threshold) or leaf.value_is_safe():
         if leaf.is_singularity() or leaf.dimensions() == 1:
             return [Bucket(tuple(leaf.bucket_intervals()), leaf.noisy_count())]
         else:

syndiffix/forest.py

@@ -2,6 +2,7 @@
 
 import math
 import random
+from typing import Optional
 
 import numpy as np
 import numpy.typing as npt
@@ -31,12 +32,17 @@ def __init__(
         counters_factory: CountersFactory,
         pids: DataFrame,
         data: DataFrame,
+        value_safe_columns_array: Optional[list[bool]] = None,
     ) -> None:
         self.anonymization_params = anonymization_params
         self.bucketization_params = bucketization_params
         self.counters_factory = counters_factory
         self.orig_pids = pids
         self.orig_data = data
+        if value_safe_columns_array is None:
+            self.value_safe_columns_array = [False] * len(data.columns)
+        else:
+            self.value_safe_columns_array = value_safe_columns_array
         self.unsafe_rng = random.Random(0)
 
         assert len(pids) == len(data)
@@ -73,7 +79,7 @@ def __init__(
         snapped_intervals = list(self.snapped_intervals)
         for i in range(self.dimensions):
             combination = (ColumnId(i),)
-            tree = self._build_tree(combination).push_down_1dim_root()
+            tree = self._build_tree(combination, self.value_safe_columns_array[i]).push_down_1dim_root()
             snapped_intervals[i] = tree.snapped_intervals[0]
             self._tree_cache[combination] = tree  # Cache the flattened version of the tree.
         self.snapped_intervals = tuple(snapped_intervals)
@@ -86,7 +92,7 @@ def _get_subnodes(self, upper_combination: Combination) -> Subnodes:
             for sub_combination in sub_combinations
         )
 
-    def _build_tree(self, combination: Combination) -> Node:
+    def _build_tree(self, combination: Combination, value_safe_flag: bool = False) -> Node:
         subnodes = self._get_subnodes(combination)
 
         # Hash column names into the tree's base bucket seed.
@@ -112,6 +118,7 @@ def _build_tree(self, combination: Combination) -> Node:
             self.bucketization_params,
             row_limit,
             self.counters_factory,
+            value_safe_flag,
         )
 
         root_intervals = get_items_combination(combination, self.snapped_intervals)

syndiffix/synthesizer.py

@@ -111,6 +111,7 @@ def __init__(
             counters_factory,
             pids,
             apply_convertors(self.column_convertors, raw_data),
+            value_safe_columns_array=self.value_safe_columns_array,
         )
 
         self.clusters, self.entropy_1dim = clustering.build_clusters(self.forest)

syndiffix/tree.py

@@ -24,6 +24,7 @@ class Context:
     bucketization_params: BucketizationParams
     row_limit: int
     counters_factory: CountersFactory
+    value_safe_flag: bool
 
 
 Subnodes = tuple[Union["Node", None], ...]
@@ -47,6 +48,9 @@ def __init__(
     def dimensions(self) -> int:
         return len(self.context.combination)
 
+    def value_is_safe(self) -> bool:
+        return self.context.value_safe_flag
+
     def update_pids(self, row: RowId) -> None:
         self.entity_counter.add(self.context.pid_data[row])
 
@@ -148,7 +152,7 @@ def _should_split(self, depth: int) -> bool:
             (depth <= depth_threshold or len(self.rows) >= self.context.row_limit)
             and not self.is_stub
             and not self.is_singularity()
-            and self.is_over_threshold(low_threshold)
+            and (self.is_over_threshold(low_threshold) or self.value_is_safe())
         )
 
     def add_row(self, depth: int, row: RowId) -> Node: