diff --git a/docs/HTML/detect_and_change.html b/docs/HTML/detect_and_change.html index 778df2a8..f27603ab 100644 --- a/docs/HTML/detect_and_change.html +++ b/docs/HTML/detect_and_change.html @@ -187,14 +187,14 @@ open_col[3000] = 2.5; open_col[5029] = 850.0; - ibm.detect_and_change<double>({ "IBM_Close", "IBM_Open" }, detect_method::fft, fill_policy::mid_point, { .threshold = 250.0, .norm_type = normalization_type::z_score, .freq_num = 1000 }); - - assert((std::fabs(close_col[502] - 82.02) < 0.01)); - assert((std::fabs(close_col[1001] - 89.805) < 0.01)); - assert((std::fabs(close_col[2002] - 88.055) < 0.01)); - assert((std::fabs(open_col[2] - 1.0) < 0.01)); // It didn't catch it - assert((std::fabs(open_col[3000] - 2.5) < 0.01)); // It didn't catch it - assert((std::fabs(open_col[5029] - 108.28) < 0.01)); + ibm.detect_and_change<double>({ "IBM_Close", "IBM_Open" }, detect_method::fft, fill_policy::mid_point, { .threshold = 80.0, .norm_type = normalization_type::none, .freq_num = 1000 }); + + assert((std::fabs(close_col[502] - 260.87) < 0.001)); + assert((std::fabs(close_col[1001] - 292.555) < 0.001)); + assert((std::fabs(close_col[2002] - 276.995) < 0.001)); + assert((std::fabs(open_col[2] - 1.0) < 0.01)); // Didn't catch this + assert((std::fabs(open_col[3000] - 210.15) < 0.01)); + assert((std::fabs(open_col[5029] - 294.488) < 0.01)); } // Now we need a DataFrame with a numeric index to be able to use @@ -226,8 +226,8 @@ assert((std::fabs(fclose_col[502] - 1.6889) < 0.0001)); assert((std::fabs(fclose_col[1001] - 1.8146) < 0.0001)); assert((std::fabs(fclose_col[2002] - 0.9022) < 0.0001)); - assert((std::fabs(fopen_col[2] - 0.01) < 0.01)); // It didn't catch it - assert((std::fabs(fopen_col[3000] - 0.05) < 0.01)); // It didn't catch it + assert((std::fabs(fopen_col[2] - 0.01) < 0.01)); // No catch + assert((std::fabs(fopen_col[3000] - 0.05) < 0.01)); // No catch assert((std::fabs(fopen_col[5029] - 7.9947) < 0.0001)); } } diff --git a/docs/HTML/remove_data_by_fft.html b/docs/HTML/remove_data_by_fft.html index a0f1b299..5aaa9e2f 100644 --- a/docs/HTML/remove_data_by_fft.html +++ b/docs/HTML/remove_data_by_fft.html @@ -128,11 +128,11 @@ assert((ibm.get_column<double>("IBM_Open").size() == 5031)); assert((ibm_view.get_column<double>("IBM_Open").size() == 5031)); - ibm.remove_data_by_fft<double, double, long>("IBM_Close", 1000, 250, normalization_type::z_score); - assert((ibm.get_column<double>("IBM_Open").size() == (5031 - 3))); + ibm.remove_data_by_fft<double, double, long>("IBM_Close", 1000, 80); + assert((ibm.get_column<double>("IBM_Open").size() == (5031 - 9))); - ibm_view.remove_data_by_fft<double, double, long>("IBM_Close", 1000, 250, normalization_type::z_score); - assert((ibm_view.get_column<double>("IBM_Open").size() == (5031 - 3))); + ibm_view.remove_data_by_fft<double, double, long>("IBM_Close", 1000, 80); + assert((ibm_view.get_column<double>("IBM_Open").size() == (5031 - 9))); } diff --git a/include/DataFrame/DataFrameMLVisitors.h b/include/DataFrame/DataFrameMLVisitors.h index 9300d898..23d7ffe3 100644 --- a/include/DataFrame/DataFrameMLVisitors.h +++ b/include/DataFrame/DataFrameMLVisitors.h @@ -1725,7 +1725,7 @@ struct EntropyVisitor { using value_type = T; using index_type = I; - using size_type = std::size_t; + using size_type = std::size_t; using result_type = typename std::conditional_t, @@ -1914,7 +1914,7 @@ struct ImpurityVisitor { using value_type = T; using index_type = I; - using size_type = std::size_t; + using size_type = std::size_t; using result_type = vec_t; template @@ -2064,7 +2064,7 @@ struct SigmoidVisitor { using value_type = T; using index_type = I; - using size_type = std::size_t; + using size_type = std::size_t; using result_type = typename std::conditional_t, @@ -2555,7 +2555,7 @@ struct PolicyLearningLossVisitor { using value_type = T; using index_type = I; - using size_type = std::size_t; + using size_type = std::size_t; using result_type = typename std::conditional_t, @@ -2708,7 +2708,7 @@ struct LossFunctionVisitor { using value_type = T; using index_type = I; - using size_type = std::size_t; + using size_type = std::size_t; using result_type = typename std::conditional_t>; @@ -3124,7 +3124,7 @@ struct VectorSimilarityVisitor { using value_type = T; using index_type = I; - using size_type = std::size_t; + using size_type = std::size_t; using result_type = double; template @@ -3491,7 +3491,7 @@ struct SpectralClusteringVisitor { // Update the accessible data // for (size_type i = 0; i < K; ++i) { - const auto val = col_s / K + 10; + const auto val = col_s / K + 10; clusters_[i].reserve(val); clusters_idxs_[i].reserve(val); @@ -3545,7 +3545,7 @@ struct SeasonalPeriodVisitor { using value_type = T; using index_type = I; - using size_type = std::size_t; + using size_type = std::size_t; using result_type = double; using md_result_type = std::vector; @@ -3788,7 +3788,7 @@ struct DynamicTimeWarpVisitor { using value_type = T; using index_type = I; - using size_type = std::size_t; + using size_type = std::size_t; using result_type = double; using distance_func = std::function; @@ -3960,7 +3960,7 @@ struct AnomalyDetectByFFTVisitor { fft.post(); // fft_res type: - // scalar --> channel_result_t = vec> + // scalar --> channel_result_t = vec> // MD --> Matrix // auto fft_res = std::move(fft.get_result()); @@ -5577,7 +5577,7 @@ struct LSTMForecastVisitor { private: - static constexpr bool is_md_ = random_acc_cont; + static constexpr bool is_md_ { random_acc_cont }; using data_t = typename std::conditional_t data_t { data_t sum { 0 }; @@ -6159,8 +6160,7 @@ struct LSTMForecastVisitor { if constexpr (! is_md_) { for (long i { 0 }; i < long(col_s); ++i) - norm_data[i][0] = - (*(column_begin + i) - mean) / std_dev; + norm_data[i][0] = (*(column_begin + i) - mean) / std_dev; } else { for (long i { 0 }; i < long(col_s); ++i) { @@ -6381,11 +6381,11 @@ struct AnomalyDetectByKNNVisitor { // KDTree::point_t is std::vector. // For the scalar path, a "window" flattened to vector is - // just {x[0], x[1], ..., x[window-1]}. + // just { x[0], x[1], ..., x[window-1] }. // For the MD path, a "window" is the concatenation of all inner elements - // across the window: {x[0][0],...,x[0][d-1], x[1][0],...,x[w-1][d-1]}. + // across the window: { x[0][0],...,x[0][d-1], x[1][0],...,x[w-1][d-1] }. // - using tree_point_t = std::vector; + using tree_point_t = std::vector; using tree_points_t = std::vector; using distance_func = @@ -6427,9 +6427,8 @@ struct AnomalyDetectByKNNVisitor { for (size_type i { 1 }; i < col_s; ++i) if ((column_begin + i)->size() != dim) - throw DataFrameError( - "AnomalyDetectByKNNVisitor: " - "Inconsistent data dimensions"); + throw DataFrameError("AnomalyDetectByKNNVisitor: " + "Inconsistent data dimensions"); } #endif // HMDF_SANITY_EXCEPTIONS @@ -6524,10 +6523,10 @@ struct AnomalyDetectByKNNVisitor { // tree_point_t (std::vector) that KDTree expects. // // Scalar path (T = double): - // value_type = double, one element per position → [v0, v1, ..., v_{w-1}] + // value_type = double, one element per position -> [v0, v1, ..., v_{w-1}] // // MD path (T = std::vector or std::array): - // Each position contributes its inner elements concatenated → + // Each position contributes its inner elements concatenated -> // [v0[0],...,v0[d-1], v1[0],...,v_{w-1}[d-1]] // template @@ -6743,9 +6742,9 @@ struct BIRCHVisitor { inline void calc_clusters_(const H &column_begin, size_type col_s) { - order_type clusters_idxs(k_); + order_type clusters_idxs(result_.size()); - for (size_type i { 0 }; i < k_; ++i) [[likely]] + for (size_type i { 0 }; i < size_type(result_.size()); ++i) [[likely]] clusters_idxs[i].reserve(col_s / k_ + 2); for (size_type j { 0 }; j < col_s; ++j) [[likely]] { @@ -6755,7 +6754,7 @@ struct BIRCHVisitor { double min_dist { std::numeric_limits::max() }; size_type min_idx { 0 }; - for (size_type i { 0 }; i < k_; ++i) { + for (size_type i { 0 }; i < size_type(result_.size()); ++i) { const double dist { dfunc_(value, result_[i]) }; if (dist < min_dist) { @@ -6789,15 +6788,19 @@ struct BIRCHVisitor { double current_threshold { threshold_ }; CFTree tree { }; - if constexpr (! is_md_) { - CFTree tmp_tree { threshold_, 1, get_dist_func_() }; + CFTree tmp_tree { + threshold_, 1, get_dist_func_(), max_entries_ + }; tree = std::move(tmp_tree); } else { CFTree tmp_tree { - threshold_, size_type(column_begin->size()), get_dist_func_() + threshold_, + size_type(column_begin->size()), + get_dist_func_(), + max_entries_ }; tree = std::move(tmp_tree); @@ -6806,6 +6809,8 @@ struct BIRCHVisitor { for (size_type i { 0 }; i < size_type(col_s); ++i) { const value_type &val { *(column_begin + i) }; + if (is_nan__(val)) continue; + while (! tree.insert(val)) { // Tree is full, rebuild with larger threshold // @@ -6891,8 +6896,8 @@ struct BIRCHVisitor { // and a known value, fit the Kriging system so you can later call predict() at // any arbitrary location and get back a value plus its uncertainty. // -template +template struct KrigingVisitor { public: @@ -6950,7 +6955,7 @@ struct KrigingVisitor { // recurrence starting from K_0 and K_1 computed through the standard // rational Chebyshev approximations (Abramowitz & Stegun 9.8). // - auto k0 = [](double t) -> double { + auto k0 = [](double t) -> double { if (t <= 2.0) { const double y { t * t / 4.0 }; @@ -6970,7 +6975,7 @@ struct KrigingVisitor { z * (-0.00251540 + z * 0.00053208))))))); } }; - auto k1 = [](double t) -> double { + auto k1 = [](double t) -> double { if (t <= 2.0) { const double y { t * t / 4.0 }; @@ -7004,7 +7009,7 @@ struct KrigingVisitor { const size_type n { static_cast(std::floor(nu)) }; if (n == 0) return (k_prev); - for (size_type i = 1; i < n; ++i) { + for (size_type i { 1 }; i < n; ++i) { const double k_next { (2.0 * i / x) * k_curr + k_prev }; k_prev = k_curr; @@ -7051,7 +7056,7 @@ struct KrigingVisitor { distance_(const coord_type &p1, const coord_type &p2) const noexcept { if constexpr (DIM == 2) { - if (params_.anisotropy_ratio < value_type(1) - value_type(1e-12)) { + if (params_.anisotropy_ratio < (T(1) - T(1e-12))) { const value_type dx { p1[0] - p2[0] }; const value_type dy { p1[1] - p2[1] }; const value_type ca { std::cos(params_.anisotropy_angle) }; @@ -7083,8 +7088,7 @@ struct KrigingVisitor { // inline value_type matern_corr_(value_type h) const { - if (h <= 0) [[unlikely]] - return (value_type(1)); + if (h <= 0) [[unlikely]] return (value_type(1)); const double nu { static_cast(params_.matern_smoothness) }; const double r { static_cast(params_.range) }; @@ -7104,8 +7108,7 @@ struct KrigingVisitor { else if (std::fabs(nu - 2.5) < 1e-9) { const double s5 { std::sqrt(5.0) * hd / r }; - return (static_cast( - (1.0 + s5 + (s5 * s5) / 3.0) * std::exp(-s5))); + return (value_type((1.0 + s5 + (s5 * s5) / 3.0) * std::exp(-s5))); } else if (nu > 50.0) { // Numerically converges to the Gaussian model in this limit. @@ -7215,8 +7218,7 @@ struct KrigingVisitor { obs_ = obs; const size_type dim_sys { params_.ordinary ? n + 1 : n }; - - matrix_t K { dim_sys, dim_sys, 0 }; + matrix_t K { dim_sys, dim_sys, 0 }; for (size_type i { 0 }; i < n; ++i) { K(i, i) = covariance_(value_type(0)) + params_.ridge; @@ -7255,16 +7257,19 @@ struct KrigingVisitor { vec_t y(sys_dim_, 0), z(sys_dim_, 0), x(sys_dim_, 0); for (size_type i { 0 }; i < sys_dim_; ++i) { - y[i] = rhs[i]; + auto &iy_val { y[i] }; + + iy_val = rhs[i]; for (size_type j { 0 }; j < i; ++j) - y[i] -= lower_(i, j) * y[j]; + iy_val -= lower_(i, j) * y[j]; + z[i] = iy_val / diag_[i]; } - for (size_type i { 0 }; i < sys_dim_; ++i) - z[i] = y[i] / diag_[i]; for (size_type i { sys_dim_ - 1 }; i >= 0; --i) { - x[i] = z[i]; + auto &ix_val { x[i] }; + + ix_val = z[i]; for (size_type j { i + 1 }; j < sys_dim_; ++j) - x[i] -= lower_(j, i) * x[j]; + ix_val -= lower_(j, i) * x[j]; } return (x); } @@ -7305,9 +7310,9 @@ struct KrigingVisitor { // The core BLUE (Best Linear Unbiased Estimator) computation against // whatever system (global or a local neighborhood) is currently cached: - // weights = K^-1 * k0 (k0 = covariances to the query point) - // value = weights . observations (+ Lagrange handling) - // variance = C(0) - k0 . weights (- mu, for Ordinary Kriging) + // weights = K^-1 * k0 (k0 = covariances to the query point) + // value = weights . observations (+ Lagrange handling) + // variance = C(0) - k0 . weights (- mu, for Ordinary Kriging) // Estimate predict_global_(const coord_type &query_pt) const { @@ -7371,9 +7376,8 @@ struct KrigingVisitor { #ifdef HMDF_SANITY_EXCEPTIONS if (col_s != size_type(std::distance(obs_begin, obs_end))) - throw DataFrameError( - "KrigingVisitor: coordinate and observation " - "columns must have equal length"); + throw DataFrameError("KrigingVisitor: coordinate and observation " + "columns must have equal length"); #endif // HMDF_SANITY_EXCEPTIONS params_.sanity_check(); @@ -7393,8 +7397,8 @@ struct KrigingVisitor { bool has_nan { false }; for (size_type d { 0 }; d < size_type(DIM); ++d) - if (is_nan__((*cit)[d])) [[unlikely]] { has_nan = true; break; } - if (has_nan) [[unlikely]] continue; + if (is_nan__((*cit)[d])) [[unlikely]] { has_nan = true; break; } + if (has_nan) [[unlikely]] continue; coords.push_back(*cit); obs.push_back(*oit); @@ -7453,9 +7457,9 @@ struct KrigingVisitor { // template inline result_type - predict_many (const QV &begin, const QV &end) const { + predict_many(const QV &begin, const QV &end) const { - result_type result(static_cast(std::distance(begin, end))); + result_type result { size_type(std::distance(begin, end)) }; size_type i { 0 }; for (auto it { begin }; it != end; ++it) @@ -7471,19 +7475,54 @@ struct KrigingVisitor { // inline vec_t loo_cross_validate () const { - const size_type n { size_type(coords_.size()) }; - vec_t residuals(n, 0); + const size_type n { size_type(coords_.size()) }; + vec_t residuals(n, 0); + const bool local { + params_.max_neighbors > 0 && + size_type(params_.max_neighbors) < (n - 1) + }; + const size_type k { + local ? size_type(params_.max_neighbors) : (n - 1) + }; + vec_t train_c; + vec_t train_o; + vec_t> dists; + train_c.reserve(local ? k : n - 1); + train_o.reserve(local ? k : n - 1); + dists.reserve(local ? n - 1 : 0); for (size_type leave_out { 0 }; leave_out < n; ++leave_out) { - vec_t train_c; - vec_t train_o; + train_c.clear(); + train_o.clear(); - train_c.reserve(n - 1); - train_o.reserve(n - 1); - for (size_type i = 0; i < n; ++i) { - if (i == leave_out) continue; - train_c.push_back(coords_[i]); - train_o.push_back(obs_[i]); + // Restrict each refit to the max_neighbors closest other points, + // mirroring predict_local_, so loo-cv stays tractable for large N + // under local-Kriging configurations (each refit becomes O(k^3) + // instead of O(n^3)) instead of silently ignoring max_neighbors. + // + if (local) { + dists.clear(); + for (size_type i { 0 }; i < n; ++i) { + if (i == leave_out) continue; + dists.push_back( + { distance_(coords_[i], coords_[leave_out]), i }); + } + std::partial_sort( + dists.begin(), dists.begin() + k, dists.end(), + [](const auto &a, const auto &b) { + return (a.first < b.first); + }); + for (size_type i { 0 }; i < k; ++i) { + train_c.push_back(coords_[dists[i].second]); + train_o.push_back(obs_[dists[i].second]); + } + } + else { + for (size_type i = 0; i < n; ++i) { + if (i == leave_out) continue; + train_c.push_back(coords_[i]); + train_o.push_back(obs_[i]); + } } KrigingVisitor loo(params_); @@ -7496,7 +7535,7 @@ struct KrigingVisitor { } return (residuals); } - + explicit KrigingVisitor(const KrigingParams ¶ms = { }) : params_(params) { } @@ -7546,7 +7585,7 @@ struct SilhouetteScoreVisitor { return ([](const T &x, const T &y) -> double { double sum { 0.0 }; - for (size_type i { 0 }; i < size_type(x.size()); ++i) { + for (size_type i { 0 }; i < size_type(x.size()); ++i) { const double diff { static_cast(x[i]) - static_cast(y[i]) @@ -7558,8 +7597,8 @@ struct SilhouetteScoreVisitor { }); } - static constexpr long NOISE_1 { -1L }; // DBSCAN unclassified sentinel - static constexpr long NOISE_2 { -2L }; // DBSCAN noise sentinel + static constexpr long NOISE_1 { -1L }; // DBSCAN unclassified sentinel + static constexpr long NOISE_2 { -2L }; // DBSCAN noise sentinel public: @@ -7567,7 +7606,7 @@ struct SilhouetteScoreVisitor { // template inline void - operator()(const K &idx_begin, const K &idx_end, + operator()(const K &idx_begin, const K &idx_end, const H1 &col_begin, const H1 &col_end, const H2 &lbl_begin, const H2 &lbl_end) { @@ -7600,18 +7639,16 @@ struct SilhouetteScoreVisitor { cluster_ids.push_back(lbl); } std::sort(cluster_ids.begin(), cluster_ids.end()); - cluster_ids.erase( - std::unique(cluster_ids.begin(), cluster_ids.end()), - cluster_ids.end()); + cluster_ids.erase(std::unique(cluster_ids.begin(), cluster_ids.end()), + cluster_ids.end()); const size_type n_clusters { cluster_ids.size() }; - // Fewer than 2 meaningful clusters → silhouette is undefined + // Fewer than 2 meaningful clusters -> silhouette is undefined // - if (n_clusters < 2) [[unlikely]] - return; + if (n_clusters < 2) [[unlikely]] return; - // Build cluster label → ordinal map for O(log k) lookup + // Build cluster label -> ordinal map for O(log k) lookup // std::vector> label_to_ord; @@ -7622,9 +7659,11 @@ struct SilhouetteScoreVisitor { auto label_ord = [&label_to_ord = std::as_const(label_to_ord)] (long lbl) -> size_type { - auto it = std::lower_bound( - label_to_ord.begin(), label_to_ord.end(), lbl, - [](const auto &p, long v) { return p.first < v; }); + const auto it { + std::lower_bound( + label_to_ord.begin(), label_to_ord.end(), lbl, + [](const auto &p, long v) { return (p.first < v); }) + }; return (it->second); }; @@ -7651,24 +7690,23 @@ struct SilhouetteScoreVisitor { // per-point, per-cluster distance sums // dist_sums[i][c] = Σⱼ∈cluster_c dist(i, j) // - std::vector> - dist_sums(col_s, std::vector(n_clusters, 0.0)); + std::vector> dist_sums( + col_s, std::vector(n_clusters, 0.0) + ); for (size_type i { 0 }; i < col_s; ++i) { const long lbl { *(lbl_begin + i) }; - if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] - continue; + if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] continue; const value_type &vi { *(col_begin + i) }; for (size_type j { 0 }; j < col_s; ++j) { - if (i == j) [[unlikely]] continue; + if (i == j) [[unlikely]] continue; const long lbl { *(lbl_begin + j) }; - if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] - continue; + if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] continue; const size_type c_j { label_ord(lbl) }; @@ -7684,13 +7722,12 @@ struct SilhouetteScoreVisitor { for (size_type i { 0 }; i < col_s; ++i) { const long lbl_i { *(lbl_begin + i) }; - if (lbl_i == NOISE_1 || lbl_i == NOISE_2) [[unlikely]] - continue; + if (lbl_i == NOISE_1 || lbl_i == NOISE_2) [[unlikely]] continue; const size_type c_i { label_ord(lbl_i) }; const size_type sz_i { cluster_sz[c_i] }; - // Singleton: a(i) undefined → s(i) = 0 + // Singleton: a(i) undefined -> s(i) = 0 // if (sz_i < 2) [[unlikely]] { ++scored_cnt; @@ -7701,11 +7738,10 @@ struct SilhouetteScoreVisitor { // b(i) = mean distance to the nearest other cluster // - double b_i { std::numeric_limits::max() }; + double b_i { std::numeric_limits::max() }; for (size_type c { 0 }; c < n_clusters; ++c) { - if (c == c_i) [[unlikely]] continue; - if (cluster_sz[c] == 0) [[unlikely]] continue; + if (c == c_i || cluster_sz[c] == 0) [[unlikely]] continue; const double mean_d { dist_sums[i][c] / double(cluster_sz[c]) @@ -7718,8 +7754,8 @@ struct SilhouetteScoreVisitor { const double denom { std::max(a_i, b_i) }; const double s_i { (denom > 0.0) ? (b_i - a_i) / denom : 0.0 }; - scores_[i] = s_i; - score_sum += s_i; + scores_[i] = s_i; + score_sum += s_i; ++scored_cnt; } @@ -7773,7 +7809,6 @@ struct DaviesBouldinIndexVisitor { template using vec_t = std::vector::type>; - // Centroid type: double for scalar T, std::vector for MD T using centroid_t = std::conditional_t>; @@ -7817,6 +7852,23 @@ struct DaviesBouldinIndexVisitor { // This ensures the user-supplied distance function is used consistently // for both intra-cluster scatter and inter-centroid separation. // + // Known limitation for non-floating-point T (e.g. int, long): centroids + // are inherently fractional (a sum divided by a count), but dfunc_'s + // signature is double(const T&, const T&), so a centroid must be cast + // back to T before every dfunc_ call here and in the scatter loop below + // -- silently truncating it. This is unavoidable without either an + // API-breaking change to distance_func's signature or dropping support + // for user-supplied custom distance functions (both scatter_ and + // centroid_dist_ must route through whatever dfunc_ the caller provided + // to honor a custom distance, and there is no way to know at runtime + // whether dfunc_ is the default or user-supplied). Callers with + // non-floating-point T should be aware DB-index/scatter values will be + // computed against truncated centroids; supplying an explicit + // floating-point-aware distance function does not avoid this, since the + // centroid itself is still cast to T before being passed in. This + // mirrors the equivalent, already-documented caveat for non-double MD + // element types below. + // inline double centroid_dist_(const centroid_t &a, const centroid_t &b) const { @@ -7849,7 +7901,7 @@ struct DaviesBouldinIndexVisitor { template inline void - operator()(const K &idx_begin, const K &idx_end, + operator()(const K &idx_begin, const K &idx_end, const H1 &col_begin, const H1 &col_end, const H2 &lbl_begin, const H2 &lbl_end) { @@ -7882,14 +7934,14 @@ struct DaviesBouldinIndexVisitor { cluster_ids.push_back(lbl); } std::sort(cluster_ids.begin(), cluster_ids.end()); - cluster_ids.erase( - std::unique(cluster_ids.begin(), cluster_ids.end()), - cluster_ids.end()); + cluster_ids.erase(std::unique(cluster_ids.begin(), cluster_ids.end()), + cluster_ids.end()); const size_type k { cluster_ids.size() }; - if (k < 2) [[unlikely]] - return; // DB undefined for fewer than 2 clusters + // DB undefined for fewer than 2 clusters + // + if (k < 2) [[unlikely]] return; // label -> ordinal (binary-search on sorted cluster_ids) // @@ -7914,7 +7966,7 @@ struct DaviesBouldinIndexVisitor { for (size_type i { 0 }; i < col_s; ++i) { const long lbl { *(lbl_begin + i) }; - if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] continue; + if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] continue; const size_type c { label_ord(lbl) }; @@ -7933,8 +7985,7 @@ struct DaviesBouldinIndexVisitor { for (size_type i { 0 }; i < col_s; ++i) { const long lbl { *(lbl_begin + i) }; - if (lbl == NOISE_1 || lbl == NOISE_2) - continue; + if (lbl == NOISE_1 || lbl == NOISE_2) continue; dim = size_type((col_begin + i)->size()); break; } @@ -7944,7 +7995,7 @@ struct DaviesBouldinIndexVisitor { for (size_type i { 0 }; i < col_s; ++i) { const long lbl { *(lbl_begin + i) }; - if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] continue; + if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] continue; const size_type c { label_ord(lbl) }; const value_type &pt { *(col_begin + i) }; @@ -7973,7 +8024,7 @@ struct DaviesBouldinIndexVisitor { for (size_type i { 0 }; i < col_s; ++i) { const long lbl { *(lbl_begin + i) }; - if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] continue; + if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] continue; const size_type c { label_ord(lbl) }; @@ -7988,7 +8039,7 @@ struct DaviesBouldinIndexVisitor { for (size_type i { 0 }; i < col_s; ++i) { const long lbl { *(lbl_begin + i) }; - if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] continue; + if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] continue; const size_type c { label_ord(lbl) }; const value_type &pt { *(col_begin + i) }; @@ -8001,7 +8052,7 @@ struct DaviesBouldinIndexVisitor { sq += diff * diff; } - scatter_[c] += std::sqrt(sq); // Euclidean, consistent + scatter_[c] += std::sqrt(sq); // Euclidean, consistent } } @@ -8049,7 +8100,7 @@ struct DaviesBouldinIndexVisitor { worst_ratio_.clear(); centroids_.clear(); } - inline void post() { } + inline void post() { } // The Davies-Bouldin index ∈ [0, ∞). Lower is better. // @@ -8118,8 +8169,14 @@ struct CalinskiHarabaszVisitor { private: - // Default: squared Euclidean for scalar (matches KMeans/DBSCAN/DB), - // Euclidean for MD. + // Default: squared Euclidean for both scalar and MD (matches + // KMeans/DBSCAN/DB's own scalar default). Calinski-Harabasz is a + // variance-ratio statistic -- BGSS and WGSS are explicitly sums of + // squares by definition (see the formula comments in operator() below) + // -- so, unlike Silhouette/Davies-Bouldin where squared-vs-raw distance + // is a free choice of metric, using raw (rooted) Euclidean distance for + // MD here would not merely rescale the result: it would stop computing + // a sum of squares at all, breaking the statistic's own definition. // inline static distance_func default_dist_() { @@ -8139,13 +8196,15 @@ struct CalinskiHarabaszVisitor { sum += diff * diff; } - return (std::sqrt(sum)); + return (sum); }); } // Distance between two centroid_t values — routes through dfunc_ for // scalar so that custom distance functions apply to centroid comparisons - // (same fix applied in DaviesBouldinIndexVisitor). + // (same fix applied in DaviesBouldinIndexVisitor). Returns squared + // distance for MD too, to match default_dist_ and the sum-of-squares + // definitions of BGSS/WGSS below -- see default_dist_'s comment. // inline double centroid_dist_(const centroid_t &a, const centroid_t &b) const { @@ -8160,7 +8219,7 @@ struct CalinskiHarabaszVisitor { sum += diff * diff; } - return (std::sqrt(sum)); + return (sum); } } @@ -8246,8 +8305,7 @@ struct CalinskiHarabaszVisitor { for (size_type i { 0 }; i < col_s; ++i) { const long lbl { *(lbl_begin + i) }; - if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] - continue; + if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] continue; const value_type v { *(col_begin + i) }; const size_type c { label_ord(lbl) }; @@ -8283,8 +8341,7 @@ struct CalinskiHarabaszVisitor { for (size_type i { 0 }; i < col_s; ++i) { const long lbl { *(lbl_begin + i) }; - if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] - continue; + if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] continue; const size_type c { label_ord(lbl) }; const value_type &pt { *(col_begin + i) }; @@ -8297,10 +8354,8 @@ struct CalinskiHarabaszVisitor { } } for (size_type c { 0 }; c < k; ++c) - for (auto &v : centroids_[c]) - v /= double(counts[c]); - for (auto &v : global_centroid_) - v /= double(clean_cnt); + for (auto &v : centroids_[c]) v /= double(counts[c]); + for (auto &v : global_centroid_) v /= double(clean_cnt); } // 3. BGSS = Σᵢ nᵢ · d²(μᵢ, μ) @@ -8319,8 +8374,7 @@ struct CalinskiHarabaszVisitor { for (size_type i { 0 }; i < col_s; ++i) { const long lbl { *(lbl_begin + i) }; - if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] - continue; + if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] continue; const size_type c { label_ord(lbl) }; @@ -8332,8 +8386,7 @@ struct CalinskiHarabaszVisitor { for (size_type i { 0 }; i < col_s; ++i) { const long lbl { *(lbl_begin + i) }; - if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] - continue; + if (lbl == NOISE_1 || lbl == NOISE_2) [[unlikely]] continue; const size_type c { label_ord(lbl) }; const value_type &pt { *(col_begin + i) }; @@ -8344,7 +8397,10 @@ struct CalinskiHarabaszVisitor { sq += diff * diff; } - cluster_wgss_[c] += std::sqrt(sq); + + // squared, matching BGSS/WGSS's own sum-of-squares definition + // + cluster_wgss_[c] += sq; } } @@ -8471,7 +8527,6 @@ struct AnomalyDetectByIsoForestVisitor { }; result_.reserve(col_s / 10 + 4); - if (nt_ > normalization_type::none) { const auto &nr { norm.get_result() }; @@ -8506,11 +8561,11 @@ struct AnomalyDetectByIsoForestVisitor { inline const scores_type &get_scores() const { return (scores_); } explicit - AnomalyDetectByIsoForestVisitor(long num_trees = 100, - long max_depth = 10, - double threshold = 0.6, - normalization_type norm_type = - normalization_type::none) + AnomalyDetectByIsoForestVisitor( + long num_trees = 100, + long max_depth = 10, + double threshold = 0.6, + normalization_type norm_type = normalization_type::none) : num_trees_(num_trees), max_depth_(max_depth), threshold_(threshold), diff --git a/include/DataFrame/Internals/DataFrame.tcc b/include/DataFrame/Internals/DataFrame.tcc index accc5e5e..d69eb287 100644 --- a/include/DataFrame/Internals/DataFrame.tcc +++ b/include/DataFrame/Internals/DataFrame.tcc @@ -2100,10 +2100,14 @@ bucketize_async(bucket_type bt, using res_t = DataFrame>; + // value must be captured by copy. It is a const reference parameter, so + // the caller can, and typically does, pass a temporary (e.g. a literal). + // That temporary is gone by the time this lambda runs on another thread. + // return (thr_pool_.dispatch( true, [bt, - &value, + value = V(value), idx_visitor = std::forward(idx_visitor), ... args = std::forward(args), this]() mutable -> res_t { diff --git a/include/DataFrame/Internals/DataFrame_functors.h b/include/DataFrame/Internals/DataFrame_functors.h index aa2e3c74..cd83d681 100644 --- a/include/DataFrame/Internals/DataFrame_functors.h +++ b/include/DataFrame/Internals/DataFrame_functors.h @@ -79,11 +79,10 @@ template struct sort_functor_ : DataVec::template visitor_base { inline sort_functor_ (const StlVecType &si, size_t is) - : sorted_idxs(si), idx_s(is), done_vec(idx_s) { } + : sorted_idxs(si), idx_s(is) { } const StlVecType &sorted_idxs; const size_t idx_s; - StlVecType done_vec; template void operator() (T2 &vec); @@ -346,16 +345,16 @@ struct equal_functor_ : DataVec::template visitor_base { template struct mod_by_idx_functor_ : DataVec::template visitor_base { + using match_vec_t = StlVecType>; + inline mod_by_idx_functor_ (const char *n, const DataFrame &d, - size_type li, - size_type ri) - : name(n), rhs_df(d), lhs_idx(li), rhs_idx(ri) { } + const match_vec_t &m) + : name(n), rhs_df(d), matches(m) { } - const char *name; - const DataFrame &rhs_df; - const size_type lhs_idx; - const size_type rhs_idx; + const char *name; + const DataFrame &rhs_df; + const match_vec_t &matches; // (lhs row, rhs row) pairs template void operator() (T &lhs_vec) const; @@ -707,14 +706,19 @@ struct columns_info_functor_ : DataVec::template visitor_base { template struct copy_remove_functor_ : DataVec::template visitor_base { + // del_mask[i] is non-zero if row i is to be removed. del_count is the + // number of such rows. + // inline copy_remove_functor_ (const char *n, - const DFUnorderedSet &td, + const StlVecType &dm, + size_type dc, DF &d) - : name(n), to_delete (td), df(d) { } + : name(n), del_mask (dm), del_count (dc), df(d) { } - const char *name; - const DFUnorderedSet &to_delete; - DF &df; + const char *name; + const StlVecType &del_mask; + const size_type del_count; + DF &df; template void operator() (const T &vec); diff --git a/include/DataFrame/Internals/DataFrame_misc.tcc b/include/DataFrame/Internals/DataFrame_misc.tcc index e4e37ac6..61822573 100644 --- a/include/DataFrame/Internals/DataFrame_misc.tcc +++ b/include/DataFrame/Internals/DataFrame_misc.tcc @@ -41,7 +41,7 @@ template template template void -DataFrame::consistent_functor_::operator() (T &vec) const { +DataFrame::consistent_functor_::operator()(T &vec) const { using ValueType = typename std::remove_reference::type::value_type; @@ -55,7 +55,7 @@ template template template void -DataFrame::shrink_to_fit_functor_::operator() (T &vec) const { +DataFrame::shrink_to_fit_functor_::operator()(T &vec) const { using value_type = typename T::value_type; @@ -73,7 +73,7 @@ template template template void -DataFrame::remove_column_functor_::operator() (T &) const { +DataFrame::remove_column_functor_::operator()(T &) const { using value_type = typename T::value_type; @@ -86,9 +86,26 @@ template template template void -DataFrame::sort_functor_::operator() (T2 &vec) { - - _sort_by_sorted_index_(vec, sorted_idxs, done_vec, idx_s); +DataFrame::sort_functor_::operator()(T2 &vec) { + + // Gather: result[i] = vec[sorted_idxs[i]]. Random reads with sequential + // writes, followed by a sequential copy back, is far more cache friendly + // than following the permutation cycles in place (random reads AND random + // writes). The column's buffer is deliberately kept (not swapped out), so + // pointers, iterators and views into the column stay valid across sort(). + // + T2 result(vec.get_allocator()); + + result.reserve(idx_s); + for (size_t i { 0 }; i < idx_s; ++i) [[likely]] { + auto &&item { vec[sorted_idxs[i]] }; + + if constexpr (std::is_same_v) + result.push_back(item); // std::vector proxy reference + else + result.push_back(std::move_if_noexcept(item)); + } + std::move(result.begin(), result.end(), vec.begin()); } // ---------------------------------------------------------------------------- @@ -97,7 +114,7 @@ template template template void -DataFrame::create_col_functor_::operator() (const T &) { +DataFrame::create_col_functor_::operator()(const T &) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; @@ -111,7 +128,7 @@ template template template void -DataFrame::load_functor_::operator() (const T &vec) { +DataFrame::load_functor_::operator()(const T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; @@ -132,7 +149,7 @@ template template template void -DataFrame::load_all_functor_::operator() (const T &vec) { +DataFrame::load_all_functor_::operator()(const T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; @@ -149,7 +166,7 @@ template template template void -DataFrame::remove_functor_::operator() (T &vec) { +DataFrame::remove_functor_::operator()(T &vec) { vec.erase(vec.begin() + begin, vec.begin() + end); } @@ -160,7 +177,7 @@ template template template void -DataFrame::truncate_functor_::operator() (T &vec) { +DataFrame::truncate_functor_::operator()(T &vec) { if (after < vec.size()) vec.erase(vec.begin() + after, vec.end()); @@ -175,7 +192,7 @@ template template void DataFrame::view_setup_functor_:: -operator() (T &vec) { +operator()(T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; @@ -201,7 +218,7 @@ template template template void -DataFrame::add_col_functor_::operator() (const T &) { +DataFrame::add_col_functor_::operator()(const T &) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; @@ -216,21 +233,20 @@ template template template void -DataFrame::print_csv_functor_::operator() (const T &vec) { +DataFrame::print_csv_functor_::operator()(const T &vec) { if (vec.empty()) return; using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; - _write_csv_df_header_(os, name, vec.size()) - << ':'; + const long vec_size { long(vec.size()) }; + const long sr { long(std::min(start_row, vec_size)) }; + const long er { long(std::min(end_row, vec_size)) }; - const long vec_size = vec.size(); - const long sr = std::min(start_row, vec_size); - const long er = std::min(end_row, vec_size); + _write_csv_df_header_(os, name, er - sr) << ':'; - for (long i = sr; i < er; ++i) + for (long i { sr }; i < er; ++i) _write_csv_df_index_(os, vec[i]) << delim; os << '\n'; @@ -243,18 +259,20 @@ template template template void -DataFrame::print_binary_functor_::operator() (const T &vec) { +DataFrame::print_binary_functor_::operator()(const T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; - char col_name[MAX_COL_NAME_SIZE]; + // Zero-filled, so the last byte written is never garbage + // + char col_name[MAX_COL_NAME_SIZE] { }; std::strncpy(col_name, name, sizeof(col_name) - 1); os.write(col_name, sizeof(col_name)); - const long local_start_row = std::min (long(vec.size()), start_row); - const long local_end_row = std::min (long(vec.size()), end_row); + const long local_start_row { std::min (long(vec.size()), start_row) }; + const long local_end_row { std::min (long(vec.size()), end_row) }; if constexpr (std::is_same_v || std::is_same_v || @@ -303,7 +321,7 @@ template template template void -DataFrame::print_json_functor_::operator() (const T &vec) { +DataFrame::print_json_functor_::operator()(const T &vec) { if (vec.empty()) return; @@ -313,16 +331,16 @@ DataFrame::print_json_functor_::operator() (const T &vec) { if (need_pre_comma) os << ",\n"; - _write_json_df_header_(os, name, vec.size()); + const long vec_size { long(vec.size()) }; + const long sr { long(std::min(start_row, vec_size)) }; + const long er { long(std::min(end_row, vec_size)) }; - const long vec_size = vec.size(); - const long sr = std::min(start_row, vec_size); - const long er = std::min(end_row, vec_size); + _write_json_df_header_(os, name, er - sr); os << "\"D\":["; - if (vec_size > 0) { - _write_json_df_index_(os, vec[start_row]); - for (long i = sr + 1; i < er; ++i) { + if (er > sr) { + _write_json_df_index_(os, vec[sr]); + for (long i { sr + 1 }; i < er; ++i) { os << ','; _write_json_df_index_(os, vec[i]); } @@ -338,7 +356,7 @@ template template template void DataFrame:: -print_csv2_header_functor_::operator() (const T &vec) { +print_csv2_header_functor_::operator()(const T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; @@ -359,7 +377,7 @@ template template template void DataFrame:: -print_csv2_data_functor_::operator() (const T &vec) { +print_csv2_data_functor_::operator()(const T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; @@ -379,7 +397,7 @@ template template void DataFrame:: -equal_functor_::operator() (const T &lhs_vec) { +equal_functor_::operator()(const T &lhs_vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; @@ -405,18 +423,27 @@ template template void DataFrame:: -mod_by_idx_functor_::operator() (T &lhs_vec) const { +mod_by_idx_functor_::operator()(T &lhs_vec) const { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; - const auto &iter = rhs_df.column_tb_.find(name); + const auto &iter { rhs_df.column_tb_.find(name) }; if (iter != rhs_df.column_tb_.end()) { - const ColumnVecType &rhs_vec = - rhs_df.get_column(name, false); - - lhs_vec[lhs_idx] = rhs_vec[rhs_idx]; + const ColumnVecType &rhs_vec { + rhs_df.get_column(name, false) + }; + const size_type lhs_s { lhs_vec.size() }; + const size_type rhs_s { rhs_vec.size() }; + + // The column lookup is done once per column, not once per row. A + // column could be shorter than the index. + // + for (const auto &[lhs_i, rhs_i] : matches) { + if (lhs_i < lhs_s && rhs_i < rhs_s) + lhs_vec[lhs_i] = rhs_vec[rhs_i]; + } } } @@ -427,7 +454,7 @@ template template void DataFrame:: -create_join_common_col_functor_::operator() (const T &) { +create_join_common_col_functor_::operator()(const T &) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; @@ -523,23 +550,25 @@ template void DataFrame:: concat_load_view_functor_:: -operator() (const T &vec) { +operator()(const T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; - const size_type rhs_s = vec.size(); - const bool has_col = result.has_column(name); + const size_type rhs_s { vec.size() }; + const bool has_col { result.has_column(name) }; if (has_col) { - auto &result_vec = - result.template get_column(name); - const size_type min_s = - std::min(result_vec.size() + rhs_s, result.get_index().size()); - size_type count { vec.size() }; + auto &result_vec { + result.template get_column(name) + }; + const size_type min_s { + std::min(result_vec.size() + rhs_s, result.get_index().size()) + }; + size_type count { result_vec.size() }; result_vec.reserve(min_s); - for (size_type i = 0; i < rhs_s && count < min_s; ++i, ++count) + for (size_type i { 0 }; i < rhs_s && count < min_s; ++i, ++count) result_vec.push_back(const_cast(&(vec[i]))); } else { @@ -549,7 +578,6 @@ operator() (const T &vec) { typename VecType::iterator>( name, { nc_vec.begin(), nc_vec.end() }); } - } // ---------------------------------------------------------------------------- @@ -589,7 +617,7 @@ template template template void DataFrame::vertical_shift_functor_:: -operator() (T &vec) const { +operator()(T &vec) const { if (sp == shift_policy::up) shift_left(vec, n); @@ -603,7 +631,7 @@ template template template void DataFrame::rotate_functor_:: -operator() (T &vec) const { +operator()(T &vec) const { if (sp == shift_policy::up) // Rotate left // There is no checking the value of n @@ -626,21 +654,26 @@ operator()(const T &lhs_vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; - const auto rhs_citer = rhs_df.column_tb_.find(col_name); + const auto rhs_citer { rhs_df.column_tb_.find(col_name) }; if (rhs_citer == rhs_df.column_tb_.end()) return; - const DataVec &rhs_hv = rhs_df.data_[rhs_citer->second]; - const auto &rhs_vec = rhs_hv.template get_vector(); - const size_type new_col_size = - std::min(std::min(lhs_vec.size(), rhs_vec.size()), new_idx.size()); + const DataVec &rhs_hv { rhs_df.data_[rhs_citer->second] }; + const auto &rhs_vec { + rhs_hv.template get_vector() + }; + const size_type new_col_size { + std::min(std::min(lhs_vec.size(), rhs_vec.size()), new_idx.size()) + }; StlVecType new_col; - auto opt = OPT { }; - size_type lcounter = 0; - size_type rcounter = 0; + auto opt = OPT{ }; + size_type lcounter { 0 }; + size_type rcounter { 0 }; new_col.reserve(new_col_size); - for (size_type idx = 0; idx < new_col_size; ) { + for (size_type idx { 0 }; idx < new_col_size && + lcounter < lhs_vec.size() && + rcounter < rhs_vec.size(); ) { if (lhs_idx[lcounter] == rhs_idx[rcounter]) { new_col.push_back(opt(lhs_vec[lcounter], rhs_vec[rcounter])); lcounter += 1; @@ -759,7 +792,7 @@ template void DataFrame:: sel_load_functor_:: -operator() (const T &vec) { +operator()(const T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; @@ -792,7 +825,7 @@ template void DataFrame:: sel_load_view_functor_:: -operator() (T &vec) { +operator()(T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; @@ -827,21 +860,21 @@ template void DataFrame:: sel_remove_functor_:: -operator() (T &vec) const { +operator()(T &vec) const { - const size_type sel_indices_s = sel_indices.size(); - const size_type vec_s = vec.size(); - size_type del_count = 0; + if constexpr (std::is_base_of, H>::value) + remove_sorted_positions_(vec, sel_indices); + else { + const size_type sel_indices_s { sel_indices.size() }; + const size_type vec_s { vec.size() }; + size_type del_count { 0 }; - for (size_type i = 0; i < sel_indices_s; ++i) { - if (sel_indices[i] < vec_s) { - if constexpr (std::is_base_of, H>::value) - vec.erase(vec.begin() + (sel_indices[i] - del_count++)); - else + for (size_type i { 0 }; i < sel_indices_s; ++i) { + if (sel_indices[i] < vec_s) vec.erase(sel_indices[i] - del_count++); + else + break; } - else - break; } } @@ -853,7 +886,7 @@ template void DataFrame:: shuffle_functor_:: -operator() (T &vec) const { +operator()(T &vec) const { std::shuffle(vec.begin(), vec.end(), g_); } @@ -865,18 +898,20 @@ template template void DataFrame:: -random_load_data_functor_::operator() (const T &vec) { +random_load_data_functor_::operator()(const T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; - const size_type vec_s = vec.size(); - const size_type n_rows = rand_indices.size(); + const size_type vec_s { vec.size() }; + const size_type n_rows { + rand_indices.size() + }; typename DF::template ColumnVecType new_vec; - size_type prev_value { 0 }; + size_type prev_value { 0 }; new_vec.reserve(n_rows); - for (size_type i = 0; i < n_rows; ++i) [[likely]] { + for (size_type i { 0 }; i < n_rows; ++i) [[likely]] { if (rand_indices[i] < vec_s) { if (i == 0 || rand_indices[i] != prev_value) new_vec.push_back(vec[rand_indices[i]]); @@ -898,19 +933,19 @@ template template template void DataFrame:: -random_load_view_functor_::operator() (const T &vec) { +random_load_view_functor_::operator()(const T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; using ViewType = typename DF::template ColumnVecType; - const size_type vec_s = vec.size(); - const size_type n_rows = rand_indices.size(); + const size_type vec_s { vec.size() }; + const size_type n_rows { rand_indices.size() }; ViewType new_vec; size_type prev_value { 0 }; new_vec.reserve(n_rows); - for (size_type i = 0; i < n_rows; ++i) [[likely]] { + for (size_type i { 0 }; i < n_rows; ++i) [[likely]] { if (rand_indices[i] < vec_s) { if (i == 0 || rand_indices[i] != prev_value) new_vec.push_back( @@ -934,7 +969,7 @@ template template template void -DataFrame::columns_info_functor_::operator() (const T &vec) { +DataFrame::columns_info_functor_::operator()(const T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; @@ -949,7 +984,7 @@ template template void DataFrame::copy_remove_functor_:: -operator() (const T &vec) { +operator()(const T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; @@ -957,9 +992,15 @@ operator() (const T &vec) { NewVecType new_vec; - new_vec.reserve(vec.size() - to_delete.size()); - for (size_type i = 0; i < vec.size(); ++i) { - if (! to_delete.contains(i)) + const size_type vec_s { vec.size() }; + const size_type mask_s { del_mask.size() }; + + // The column could be shorter than the index (and the delete count is + // based on the index) + // + new_vec.reserve(vec_s > del_count ? vec_s - del_count : 0); + for (size_type i { 0 }; i < vec_s; ++i) { + if (i >= mask_s || ! del_mask[i]) new_vec.push_back(vec[i]); } @@ -975,18 +1016,23 @@ template template template void -DataFrame::fill_missing_functor_::operator() (T &vec) { +DataFrame::fill_missing_functor_::operator()(T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; try { - const auto &rhs_vec = - rhs.template get_column(col_name, false); - const size_type col_s = std::min( - { vec.size(), rhs_vec.size(), self_idx.size(), rhs_idx.size() }); - - for (size_type i = 0; i < col_s; ++i) + const auto &rhs_vec { + rhs.template get_column(col_name, false) + }; + const size_type col_s { + std::min({ vec.size(), + rhs_vec.size(), + self_idx.size(), + rhs_idx.size() }) + }; + + for (size_type i { 0 }; i < col_s; ++i) if (is_nan(vec[i]) && self_idx[i] == rhs_idx[i]) vec[i] = rhs_vec[i]; } @@ -999,9 +1045,9 @@ template template template void -DataFrame::describe_functor_::operator() (const T &vec) { +DataFrame::describe_functor_::operator()(const T &vec) { - const size_type vec_s = vec.size(); + const size_type vec_s { vec.size() }; if (vec_s < 3) return; @@ -1013,25 +1059,33 @@ DataFrame::describe_functor_::operator() (const T &vec) { col_to_load.reserve(describe_index_col.size()); col_to_load.push_back(double(vec_s)); - size_type missing_cnt = 0; - ValueType minv = vec[0]; - ValueType maxv = vec[0]; - ValueType sum = vec[0]; + size_type missing_cnt { 0 }; + ValueType minv { get_nan() }; + ValueType maxv { get_nan() }; + double sum { get_nan() }; + bool found_valid { false }; - if (is_nan(vec[0])) missing_cnt += 1; - for (size_type i = 1; i < vec_s; ++i) { + for (size_type i { 0 }; i < vec_s; ++i) { if (is_nan(vec[i])) missing_cnt += 1; + else if (! found_valid) { + minv = maxv = vec[i]; + sum = double(vec[i]); + found_valid = true; + } else { if (vec[i] > maxv) maxv = vec[i]; if (vec[i] < minv) minv = vec[i]; - sum += vec[i]; + sum += double(vec[i]); } } col_to_load.push_back(double(missing_cnt)); col_to_load.push_back(sum / double(vec_s - missing_cnt)); - StdVisitor stdev; + // The visitors are instantiated on double so that integral and float + // columns are not accumulated (or interpolated) in their narrow type. + // + StdVisitor stdev; stdev.pre(); stdev(vec.begin(), vec.end(), vec.begin(), vec.end()); @@ -1041,21 +1095,21 @@ DataFrame::describe_functor_::operator() (const T &vec) { col_to_load.push_back(double(minv)); col_to_load.push_back(double(maxv)); - QuantileVisitor qt25(0.25); + QuantileVisitor qt25 { 0.25 }; qt25.pre(); qt25(vec.begin(), vec.end(), vec.begin(), vec.end()); qt25.post(); col_to_load.push_back(double(qt25.get_result())); - QuantileVisitor qt50(0.5); + QuantileVisitor qt50 { 0.5 }; qt50.pre(); qt50(vec.begin(), vec.end(), vec.begin(), vec.end()); qt50.post(); col_to_load.push_back(double(qt50.get_result())); - QuantileVisitor qt75(0.75); + QuantileVisitor qt75 { 0.75 }; qt75.pre(); qt75(vec.begin(), vec.end(), vec.begin(), vec.end()); @@ -1075,28 +1129,31 @@ template template void DataFrame::change_freq_functor_:: -operator() (const T &vec) { +operator()(const T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; using NewVecType = ColumnVecType; - const auto &new_idx = res.get_index(); - const auto new_s = new_idx.size(); - const auto old_s = std::min(old_idx.size(), vec.size()); - NewVecType new_vec (new_s, get_nan()); + const auto &new_idx { res.get_index() }; + const auto new_s { new_idx.size() }; + const auto old_s { std::min(old_idx.size(), vec.size()) }; + NewVecType new_vec(new_s, get_nan()); size_type i { 0 }, j { 0 }; if (new_s >= old_s) { // Frequency has increased - while (i < new_s && j < (old_s - 1)) { + // Note: old_s is unsigned, so "old_s - 1" must not be used when old_s + // could be zero. An empty column stays all NaN. + // + while (i < new_s && (j + 1) < old_s) { if (new_idx[i] < old_idx[j + 1]) new_vec[i] = vec[j]; else new_vec[i] = vec[++j]; i += 1; } - if (i > 0) { - const ValueType &last = new_vec[i - 1]; + if (old_s > 0) { + const ValueType &last = vec[old_s - 1]; for ( ; i < new_s; ++i) new_vec[i] = last; @@ -1135,7 +1192,7 @@ template template void DataFrame::dup_mask_functor_:: -operator() (const T &vec) { +operator()(const T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; @@ -1198,16 +1255,20 @@ template template void DataFrame::explode_functor_:: -operator() (const T &vec) { +operator()(const T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; - const size_type total_cnt = idx_mask.size(); + const size_type total_cnt { idx_mask.size() }; + const size_type vec_sz { vec.size() }; VecType new_col(total_cnt); - for (size_type i { 0 }; i < total_cnt; ++i) - new_col[i] = vec[idx_mask[i]]; + for (size_type i { 0 }; i < total_cnt; ++i) { + const auto idx { idx_mask[i] }; + + if (idx < vec_sz) [[likely]] new_col[i] = vec[idx_mask[i]]; + } res.template load_column(name, std::move(new_col), @@ -1222,15 +1283,16 @@ template template void DataFrame::difference_functor_:: -operator() (const T &vec) { +operator()(const T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; try { - const auto &other_vec = - other.template get_column(name, false); - const size_type col_s = std::min(vec.size(), other_vec.size()); + const auto &other_vec { + other.template get_column(name, false) + }; + const size_type col_s { std::min(vec.size(), other_vec.size()) }; size_type last_idx { 0 }; size_type i { 0 }; VecType self_new_col; @@ -1251,11 +1313,27 @@ operator() (const T &vec) { idx_set.insert(i); } } - for (size_type j = i; j < vec.size(); ++j) [[unlikely]] { + + // The two columns could be of different lengths. The extra rows of + // the longer one (the tail) must stay aligned with the rows they + // belong to. So, the equal rows between the last difference and the + // tail must be padded first, the same way the equal rows between + // two differences are padded above. + // + if (vec.size() > col_s || other_vec.size() > col_s) { + for ( ; last_idx < col_s; ++last_idx) { + if (vec.size() > col_s) + self_new_col.push_back(get_nan()); + if (other_vec.size() > col_s) + other_new_col.push_back(get_nan()); + idx_set.insert(last_idx); + } + } + for (size_type j { i }; j < vec.size(); ++j) [[unlikely]] { self_new_col.push_back(vec[j]); idx_set.insert(j); } - for (size_type j = i; j < other_vec.size(); ++j) [[unlikely]] { + for (size_type j { i }; j < other_vec.size(); ++j) [[unlikely]] { other_new_col.push_back(other_vec[j]); idx_set.insert(j); } @@ -1289,7 +1367,7 @@ template template void DataFrame::stringfy_functor_:: -operator() (const T &vec) { +operator()(const T &vec) { using VecType = typename std::remove_reference::type; using ValueType = typename VecType::value_type; diff --git a/include/DataFrame/Internals/DataFrame_opt.tcc b/include/DataFrame/Internals/DataFrame_opt.tcc index 913073d3..207f248a 100644 --- a/include/DataFrame/Internals/DataFrame_opt.tcc +++ b/include/DataFrame/Internals/DataFrame_opt.tcc @@ -79,26 +79,32 @@ modify_by_idx (DataFrame &rhs, sort_state already_sorted) { const size_type lhs_s { indices_.size() }; const size_type rhs_s { rhs.indices_.size() }; - const SpinGuard guard(lock_); + typename mod_by_idx_functor_::match_vec_t matches; - for (size_type lhs_i = 0, rhs_i = 0; + // Find the matching rows first. The bounds of self's index must be checked + // before it is read. + // + matches.reserve(std::min(lhs_s, rhs_s)); + for (size_type lhs_i { 0 }, rhs_i { 0 }; lhs_i < lhs_s && rhs_i < rhs_s; ++rhs_i) [[likely]] { - while (indices_[lhs_i] < rhs.indices_[rhs_i] && lhs_i < lhs_s) + while (lhs_i < lhs_s && indices_[lhs_i] < rhs.indices_[rhs_i]) lhs_i += 1; + if (lhs_i >= lhs_s) break; if (indices_[lhs_i] == rhs.indices_[rhs_i]) { - for (const auto &[name, idx] : column_list_) [[likely]] { - mod_by_idx_functor_ functor (name.c_str(), - rhs, - lhs_i, - rhs_i); + matches.emplace_back(lhs_i, rhs_i); + lhs_i += 1; + } + } - data_[idx].change(functor); - } + if (! matches.empty()) { + const SpinGuard guard { lock_ }; - lhs_i += 1; + for (const auto &[name, idx] : column_list_) [[likely]] { + mod_by_idx_functor_ functor (name.c_str(), rhs, matches); + + data_[idx].change(functor); } - else if (indices_[lhs_i] < rhs.indices_[rhs_i]) break; } return (*this); diff --git a/include/DataFrame/Internals/DataFrame_private_decl.h b/include/DataFrame/Internals/DataFrame_private_decl.h index ee874a03..658f3494 100644 --- a/include/DataFrame/Internals/DataFrame_private_decl.h +++ b/include/DataFrame/Internals/DataFrame_private_decl.h @@ -692,6 +692,35 @@ fill_missing_linter_(ColumnVecType &, const IndexVecType &, int) { // ---------------------------------------------------------------------------- +// Removes the elements at the positions listed in sel_indices from vec in a +// single O(n) pass. sel_indices must be sorted in ascending order. Positions +// that are beyond the end of vec are ignored. This is only for real vectors +// (not views), since it assigns elements. +// +template +static inline void +remove_sorted_positions_(VEC &vec, const StlVecType &sel_indices) { + + const size_type vec_s { vec.size() }; + const size_type sel_s { sel_indices.size() }; + + if (sel_s == 0 || sel_indices[0] >= vec_s) return; + + size_type next { 0 }; + size_type w { sel_indices[0] }; + + for (size_type r { w }; r < vec_s; ++r) { + if (next < sel_s && sel_indices[next] == r) { + while (next < sel_s && sel_indices[next] == r) ++next; + continue; + } + vec[w++] = std::move(vec[r]); + } + vec.erase(vec.begin() + w, vec.end()); +} + +// ---------------------------------------------------------------------------- + template void remove_data_by_sel_common_(const StlVecType &col_indices) { @@ -708,20 +737,19 @@ void remove_data_by_sel_common_(const StlVecType &col_indices) { const sel_remove_functor_ functor { col_indices }; for (auto citer { begin }; citer < end; ++citer) - this->data_[citer->second].change(functor); + data_[citer->second].change(functor); }; auto lbd_idx = [&col_indices = std::as_const(col_indices), this] () -> void { - const size_type col_indices_s { col_indices.size() }; - size_type del_count { 0 }; - - for (size_type i { 0 }; i < col_indices_s; ++i) { - if constexpr ( - std::is_base_of, H>::value) - this->indices_.erase(this->indices_.begin() + - (col_indices[i] - del_count++)); - else - this->indices_.erase(col_indices[i] - del_count++); + if constexpr ( + std::is_base_of, H>::value) + remove_sorted_positions_(indices_, col_indices); + else { + const size_type col_indices_s { col_indices.size() }; + size_type del_count { 0 }; + + for (size_type i { 0 }; i < col_indices_s; ++i) + indices_.erase(col_indices[i] - del_count++); } }; auto futures { @@ -741,14 +769,13 @@ void remove_data_by_sel_common_(const StlVecType &col_indices) { data_[citer.second].change(functor); guard.release(); - const size_type col_indices_s { col_indices.size() }; - size_type del_count { 0 }; + if constexpr (std::is_base_of, H>::value) + remove_sorted_positions_(indices_, col_indices); + else { + const size_type col_indices_s { col_indices.size() }; + size_type del_count { 0 }; - for (size_type i { 0 }; i < col_indices_s; ++i) { - if constexpr (std::is_base_of, H>::value) - indices_.erase(indices_.begin() + - (col_indices[i] - del_count++)); - else + for (size_type i { 0 }; i < col_indices_s; ++i) indices_.erase(col_indices[i] - del_count++); } } @@ -1221,41 +1248,47 @@ remove_dups_common_(const DataFrame &s_df, const MAP &row_table, const IndexVecType &index) { - using count_set = DFUnorderedSet; using res_t = DataFrame>; - const size_type idx_s = index.size(); - count_set rows_to_del; + // Every row appears in exactly one of the row_table's index vectors. So, + // a plain per-row mask is both exact and much cheaper than a hash set, + // since each column is then scanned with no hashing at all. + // + const size_type idx_s { index.size() }; + StlVecType del_mask(idx_s, 0); + size_type del_count { 0 }; + auto mark = + [&del_mask, &del_count](auto first, auto last) -> void { + for ( ; first != last; ++first) { + del_mask[*first] = 1; + del_count += 1; + } + }; - rows_to_del.reserve(idx_s / 100); if (rds == remove_dup_spec::keep_first) { for (const auto &[val_tuple, idx_vec] : row_table) { - if (idx_vec.size() > 1) - rows_to_del.insert(idx_vec.begin() + 1, idx_vec.end()); + if (idx_vec.size() > 1) mark(idx_vec.begin() + 1, idx_vec.end()); } } else if (rds == remove_dup_spec::keep_last) { for (const auto &[val_tuple, idx_vec] : row_table) { - if (idx_vec.size() > 1) - rows_to_del.insert(idx_vec.begin(), idx_vec.end() - 1); + if (idx_vec.size() > 1) mark(idx_vec.begin(), idx_vec.end() - 1); } } else { // remove_dup_spec::keep_none for (const auto &[val_tuple, idx_vec] : row_table) - if (idx_vec.size() > 1) - rows_to_del.insert(idx_vec.begin(), idx_vec.end()); + if (idx_vec.size() > 1) mark(idx_vec.begin(), idx_vec.end()); } res_t new_df; typename res_t::IndexVecType new_index; - const SpinGuard guard(lock_); + const SpinGuard guard { lock_ }; // Load the index // - new_index.reserve(idx_s - rows_to_del.size()); - for (size_type i = 0; i < idx_s; ++i) { - if (! rows_to_del.contains(i)) - new_index.push_back(index[i]); + new_index.reserve(idx_s - del_count); + for (size_type i { 0 }; i < idx_s; ++i) { + if (! del_mask[i]) new_index.push_back(index[i]); } new_df.load_index(std::move(new_index)); @@ -1263,42 +1296,46 @@ remove_dups_common_(const DataFrame &s_df, // Create the columns, so loading can proceed in parallel // for (const auto &citer : s_df.column_list_) { - create_col_functor_ functor( - citer.first.c_str(), new_df); + create_col_functor_ functor { + citer.first.c_str(), new_df + }; s_df.data_[citer.second].change(functor); } - const auto thread_level = + const auto thread_level { (new_df.get_index().size() < ThreadPool::MUL_THR_THHOLD) - ? 0L : get_thread_level(); + ? 0L : get_thread_level() + }; if (thread_level > 2) { auto lbd = - [&rows_to_del = std::as_const(rows_to_del), + [&del_mask = std::as_const(del_mask), + del_count, &s_df = std::as_const(s_df), &new_df] (const auto &begin, const auto &end) -> void { - for (auto citer = begin; citer < end; ++citer) { - copy_remove_functor_ functor( - citer->first.c_str(), - rows_to_del, - new_df); + for (auto citer { begin }; citer < end; ++citer) { + copy_remove_functor_ functor { + citer->first.c_str(), del_mask, del_count, new_df + }; s_df.data_[citer->second].change(functor); } }; - auto futures = + auto futures { thr_pool_.parallel_loop(s_df.column_list_.begin(), s_df.column_list_.end(), - std::move(lbd)); + std::move(lbd)) + }; for (auto &fut : futures) fut.get(); } else { for (const auto &citer : s_df.column_list_) { copy_remove_functor_ functor(citer.first.c_str(), - rows_to_del, + del_mask, + del_count, new_df); s_df.data_[citer.second].change(functor); @@ -2463,6 +2500,70 @@ get_scaled_data_matrix_(std::vector &&col_names, // ---------------------------------------------------------------------------- +// It returns the first n entries (or all of them, if n is greater than the +// column length) of what permutation_vec(name, dir) would return. If n is +// small compared to the column, there is no need to sort the whole column. A +// partial sort is much faster (O(n log n) instead of O(n log n)), and the +// result is the same up to the order of equal values, which is not defined by +// sort() anyway. +// +template +StlVecType +n_permutation_vec_(const char *name, sort_spec dir, size_type n) const { + + const ColumnVecType *vec { nullptr }; + + { + const SpinGuard guard { lock_ }; + + if (! ::strcmp(name, DF_INDEX_COL_NAME)) + vec = reinterpret_cast *>(&indices_); + else + vec = &(get_column(name, false)); + } + + const size_type col_s { vec->size() }; + const size_type k { std::min(n, col_s) }; + + // For a large n, the full (and possibly parallel) sort is as good or + // better + // + if (k > (col_s / 32)) { + const auto full = permutation_vec(name, dir); + StlVecType result(std::min(k, full.size())); + + std::copy_n(full.begin(), result.size(), result.begin()); + return (result); + } + + StlVecType result(col_s); + auto part_sort = + [&result, k](auto &&comp) -> void { + std::partial_sort(result.begin(), result.begin() + k, + result.end(), comp); + }; + const auto &v = *vec; + + std::iota(result.begin(), result.end(), 0); + if (dir == sort_spec::ascen) + part_sort([&v](size_type l, size_type r) { return (v[l] < v[r]); }); + else if (dir == sort_spec::desce) + part_sort([&v](size_type l, size_type r) { return (v[l] > v[r]); }); + else if (dir == sort_spec::abs_ascen) + part_sort([&v](size_type l, size_type r) { + return (abs__(v[l]) < abs__(v[r])); + }); + else if (dir == sort_spec::abs_desce) + part_sort([&v](size_type l, size_type r) { + return (abs__(v[l]) > abs__(v[r])); + }); + + result.resize(k); + return (result); +} + +// ---------------------------------------------------------------------------- + // Local Variables: // mode:C++ // tab-width:4 diff --git a/include/DataFrame/Internals/DataFrame_remove.tcc b/include/DataFrame/Internals/DataFrame_remove.tcc index 017b0ca4..4b307e08 100644 --- a/include/DataFrame/Internals/DataFrame_remove.tcc +++ b/include/DataFrame/Internals/DataFrame_remove.tcc @@ -42,24 +42,34 @@ void DataFrame::remove_data_by_idx (Index2D range) { static_assert(std::is_base_of, H>::value, "Only a StdDataFrame can call remove_data_by_idx()"); - const auto &lower = - std::lower_bound (indices_.begin(), indices_.end(), range.begin); - const auto &upper = - std::upper_bound (indices_.begin(), indices_.end(), range.end); - - if (lower != indices_.end()) [[likely]] { - const size_type b_dist = std::distance(indices_.begin(), lower); - const size_type e_dist = - std::distance(indices_.begin(), - upper < indices_.end() ? upper : indices_.end()); + const auto &lower { + std::lower_bound (indices_.begin(), indices_.end(), range.begin) + }; + const auto &upper { + std::upper_bound (indices_.begin(), indices_.end(), range.end) + }; + + // If range.begin > range.end, lower could be after upper. That is an empty + // range, and erasing [lower, upper) with lower > upper is undefined. + // + if (lower != indices_.end() && lower <= upper) [[likely]] { + const size_type b_dist { + size_type(std::distance(indices_.begin(), lower)) + }; + const size_type e_dist { + size_type(std::distance(indices_.begin(), + upper < indices_.end() + ? upper : indices_.end())) + }; make_consistent(); indices_.erase(lower, upper); - const auto thread_level = + const auto thread_level { (indices_.size() < ThreadPool::MUL_THR_THHOLD) - ? 0L : get_thread_level(); - const SpinGuard guard(lock_); + ? 0L : get_thread_level() + }; + const SpinGuard guard { lock_ }; if (thread_level > 2) { auto lbd = @@ -67,13 +77,14 @@ void DataFrame::remove_data_by_idx (Index2D range) { (const auto &begin, const auto &end) -> void { remove_functor_ functor (b_dist, e_dist); - for (auto citer = begin; citer < end; ++citer) + for (auto citer { begin }; citer < end; ++citer) this->data_[citer->second].change(functor); }; - auto futures = + auto futures { thr_pool_.parallel_loop(column_list_.begin(), column_list_.end(), - std::move(lbd)); + std::move(lbd)) + }; for (auto &fut : futures) fut.get(); } @@ -500,13 +511,19 @@ remove_data_by_stdev(const char *col_name, T above_stdev, T below_stdev) { "Only a StdDataFrame or a PtrView can call " "remove_data_by_stdev()"); - const ColumnVecType &vec = get_column(col_name); - const size_type col_s = vec.size(); - const auto thread_level = - (col_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level(); + const ColumnVecType &vec { get_column(col_name) }; + const size_type col_s { vec.size() }; + const auto thread_level { + (col_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level() + }; + + // mean, stdev, and the z-score are all computed in double. If T is an + // integral type, doing this in T truncates the mean and the stdev, and a + // stdev of zero (a constant column) would be an integer division by zero. + // auto mean_lbd = - [&vec = std::as_const(vec), this]() -> T { - MeanVisitor mean { true }; + [&vec = std::as_const(vec), this]() -> double { + MeanVisitor mean { true }; mean.pre(); mean(indices_.begin(), indices_.end(), vec.begin(), vec.end()); @@ -514,20 +531,20 @@ remove_data_by_stdev(const char *col_name, T above_stdev, T below_stdev) { return (mean.get_result()); }; auto stdev_lbd = - [&vec = std::as_const(vec), this]() -> T { - StdVisitor stdev { true, true }; + [&vec = std::as_const(vec), this]() -> double { + StdVisitor stdev { true, true }; stdev.pre(); stdev(indices_.begin(), indices_.end(), vec.begin(), vec.end()); stdev.post(); return (stdev.get_result()); }; - T stdev; - T mean; + double stdev; + double mean; if (thread_level > 2) { - auto stdev_fut = thr_pool_.dispatch(false, stdev_lbd); - auto mean_fut = thr_pool_.dispatch(false, mean_lbd); + auto stdev_fut { thr_pool_.dispatch(false, stdev_lbd) }; + auto mean_fut { thr_pool_.dispatch(false, mean_lbd) }; mean = mean_fut.get(); stdev = stdev_fut.get(); @@ -540,10 +557,10 @@ remove_data_by_stdev(const char *col_name, T above_stdev, T below_stdev) { StlVecType col_indices; col_indices.reserve(col_s / 3); - for (size_type i = 0; i < col_s; ++i) { - const T z = (vec[i] - mean) / stdev; + for (size_type i { 0 }; i < col_s; ++i) { + const double z { (double(vec[i]) - mean) / stdev }; - if (z > above_stdev || z < below_stdev) + if (z > double(above_stdev) || z < double(below_stdev)) col_indices.push_back(i); } @@ -643,7 +660,7 @@ remove_data_by_isof(const char *col_name, long num_trees, long max_depth, double threshold, - normalization_type ntype) { + normalization_type ntype) { static_assert(std::is_base_of, H>::value || std::is_base_of, H>::value, diff --git a/include/DataFrame/Internals/DataFrame_set.tcc b/include/DataFrame/Internals/DataFrame_set.tcc index 2baf1afe..2f2fee32 100644 --- a/include/DataFrame/Internals/DataFrame_set.tcc +++ b/include/DataFrame/Internals/DataFrame_set.tcc @@ -47,8 +47,8 @@ DataFrame::create_column (const char *name, bool do_lock) { "Data column name cannot be 'INDEX'"); const SpinGuard guard { do_lock ? lock_ : nullptr }; - const auto iter { column_tb_.find(name) }; - + const auto iter { column_tb_.find(name) }; + if (iter != column_tb_.end()) [[unlikely]] { DataVec &hv { data_[iter->second] }; @@ -547,35 +547,66 @@ load_indicators(const char *cat_col_name, const char *numeric_cols_prefix) { static_assert(std::is_base_of, H>::value, "Only a StdDataFrame can call load_indicators()"); - using map_t = DFUnorderedMap *>; + using map_t = DFUnorderedMap; - const SpinGuard guard(lock_); - const auto &cat_col = get_column(cat_col_name, false); - const auto col_s = cat_col.size(); - map_t val_map; - size_type ret_cnt = 0; + size_type col_s { 0 }; + map_t val_map; + StlVecType unique_vals; + // The ordinal of each row's category in unique_vals + // + StlVecType row_ords; + const SpinGuard guard { lock_ }; + + // First pass: find the unique categories, in order of first appearance. + // Nothing is created in this pass. + // + { + const auto &cat_col { get_column(cat_col_name, false) }; + + col_s = cat_col.size(); + val_map.reserve(col_s / 2); + row_ords.reserve(col_s); + for (size_type i { 0 }; i < col_s; ++i) [[likely]] { + const auto in_ret { + val_map.try_emplace(cat_col[i], unique_vals.size()) + }; - val_map.reserve(col_s / 2); - for (size_type i = 0; i < col_s; ++i) [[likely]] { - const auto val = cat_col[i]; - auto in_ret = val_map.emplace(std::make_pair(val, nullptr)); + if (in_ret.second) + unique_vals.push_back(cat_col[i]); + row_ords.push_back(in_ret.first->second); + } + } - if (in_ret.second) { - ColNameType new_name; + // Second pass: create all the new columns. Creating a column can + // reallocate the column storage, which invalidates every reference to a + // column obtained before it. So, here nothing is kept but the names. + // + const size_type uniq_s { unique_vals.size() }; + StlVecType new_names; - if (numeric_cols_prefix) - new_name = numeric_cols_prefix; - new_name += _to_string_(val).c_str(); + new_names.reserve(uniq_s); + for (size_type i { 0 }; i < uniq_s; ++i) { + ColNameType new_name; - auto *new_col = &(create_column(new_name.c_str(), false)); + if (numeric_cols_prefix) + new_name = numeric_cols_prefix; + new_name += _to_string_(unique_vals[i]).c_str(); - new_col->resize(col_s, IT(0)); - in_ret.first->second = new_col; - ret_cnt += col_s; - } - in_ret.first->second->at(i) = IT(1); + create_column(new_name.c_str(), false).assign(col_s, IT(0)); + new_names.push_back(std::move(new_name)); } - return (ret_cnt); + + // Third pass: now that the column storage is stable, get the columns and + // fill them. + // + StlVecType *> new_cols(uniq_s, nullptr); + + for (size_type i { 0 }; i < uniq_s; ++i) + new_cols[i] = &(get_column(new_names[i].c_str(), false)); + for (size_type i { 0 }; i < col_s; ++i) [[likely]] + (*(new_cols[row_ords[i]]))[i] = IT(1); + + return (uniq_s * col_s); } // ---------------------------------------------------------------------------- @@ -591,22 +622,37 @@ from_indicators(const StlVecType &ind_col_names, static_assert(std::is_base_of, H>::value, "Only a StdDataFrame can call from_indicators()"); - const size_type ind_col_s = ind_col_names.size(); + const size_type ind_col_s { ind_col_names.size() }; + + if (ind_col_s == 0) [[unlikely]] + throw DataFrameError("DataFrame::from_indicators(): ERROR: " + "No indicator column name is given"); + StlVecType *> ind_cols(ind_col_s, nullptr); - SpinGuard guard (lock_); + SpinGuard guard { lock_ }; + + // Make sure all the indicator columns exist before creating the new column + // + for (size_type i { 0 }; i < ind_col_s; ++i) [[likely]] + ind_cols[i] = &(get_column(ind_col_names[i], false)); + + auto &new_col { create_column(cat_col_name, false) }; + const size_type pre_offset { + numeric_cols_prefix == nullptr ? 0 : strlen(numeric_cols_prefix) + }; - for (size_type i = 0; i < ind_col_s; ++i) [[likely]] + // Creating a column can reallocate the column storage, which invalidates + // every reference to a column obtained before it. So, get them again. + // + for (size_type i { 0 }; i < ind_col_s; ++i) [[likely]] ind_cols[i] = &(get_column(ind_col_names[i], false)); - const size_type col_s = ind_cols[0]->size(); - auto &new_col = create_column(cat_col_name, false); - const size_type pre_offset = - numeric_cols_prefix == nullptr ? 0 : strlen(numeric_cols_prefix); + const size_type col_s { ind_cols[0]->size() }; guard.release(); new_col.reserve(col_s); - for (size_type i = 0; i < col_s; ++i) [[likely]] { - for (size_type j = 0; j < ind_col_s; ++j) [[likely]] { + for (size_type i { 0 }; i < col_s; ++i) [[likely]] { + for (size_type j { 0 }; j < ind_col_s; ++j) [[likely]] { if (ind_cols[j]->at(i)) { new_col.push_back( _string_to_(ind_col_names[j] + pre_offset)); @@ -895,10 +941,10 @@ template void DataFrame:: truncate(IndexType &&before, IndexType &&after) { - const auto begin = indices_.begin(); - const auto end = indices_.end(); - const auto l_citer = std::lower_bound(begin, end, before); - const auto u_citer = std::upper_bound(begin, end, after); + const auto begin { indices_.begin() }; + const auto end { indices_.end() }; + const auto l_citer { std::lower_bound(begin, end, before) }; + const auto u_citer { std::upper_bound(begin, end, after) }; size_type l_index { 0 }; size_type u_index { indices_.size() }; @@ -907,22 +953,30 @@ truncate(IndexType &&before, IndexType &&after) { if (u_citer != end) u_index = std::distance(begin, u_citer); + // An empty or inverted range (i.e. before > after) leaves nothing. Without + // this, l_index > u_index and the second erase() of indices_ below goes + // past the end of the container. + // + if (l_index > u_index) l_index = u_index; + { - const auto thread_level = + const auto thread_level { (indices_.size() < ThreadPool::MUL_THR_THHOLD) - ? 0L : get_thread_level(); + ? 0L : get_thread_level() + }; auto lbd = [this, l_index, u_index] (auto begin, auto end) -> void { - truncate_functor_ functor (l_index, u_index); + truncate_functor_ functor { l_index, u_index }; - for (auto citer = begin; citer < end; ++citer) + for (auto citer { begin }; citer < end; ++citer) this->data_[citer->second].change(functor); }; - const SpinGuard guard(lock_); + const SpinGuard guard { lock_ }; if (thread_level > 2) { - auto futures = thr_pool_.parallel_loop( - column_list_.begin(), column_list_.end(), std::move(lbd)); + auto futures { thr_pool_.parallel_loop( + column_list_.begin(), column_list_.end(), std::move(lbd)) + }; for (auto &fut : futures) fut.get(); } diff --git a/include/DataFrame/Internals/DataFrame_shift.tcc b/include/DataFrame/Internals/DataFrame_shift.tcc index 66d90838..507c39db 100644 --- a/include/DataFrame/Internals/DataFrame_shift.tcc +++ b/include/DataFrame/Internals/DataFrame_shift.tcc @@ -44,36 +44,39 @@ void DataFrame::self_shift(size_type periods, shift_policy sp) { if (periods > 0) [[likely]] { if (sp == shift_policy::down || sp == shift_policy::up) [[likely]] { vertical_shift_functor_ functor(periods, sp); - const size_type num_cols = data_.size(); - const auto thread_level = + const size_type num_cols { data_.size() }; + const auto thread_level { (indices_.size() < ThreadPool::MUL_THR_THHOLD) - ? 0L : get_thread_level(); - const SpinGuard guard(lock_); + ? 0L : get_thread_level() + }; + const SpinGuard guard { lock_ }; if (thread_level > 2) { auto lbd = [&functor, this](auto begin, auto end) -> void { - for (size_type idx = begin; idx < end; ++idx) + for (size_type idx { begin }; idx < end; ++idx) this->data_[idx].change(functor); }; - auto futuers = + auto futuers { thr_pool_.parallel_loop( - size_type(0), num_cols, std::move(lbd)); + size_type(0), num_cols, std::move(lbd)) + }; for (auto &fut : futuers) fut.get(); } else { - for (size_type idx = 0; idx < num_cols; ++idx) [[likely]] + for (size_type idx { 0 }; idx < num_cols; ++idx) [[likely]] data_[idx].change(functor); } } else { - while (periods-- > 0) { - const char *col_name = + while (periods-- > 0 && (! column_list_.empty())) { + const char *col_name { (sp == shift_policy::left) ? column_list_.front().first.c_str() - : column_list_.back().first.c_str(); - remove_column_functor_ functor (col_name, *this); + : column_list_.back().first.c_str() + }; + remove_column_functor_ functor { col_name, *this }; for (const auto &[name, idx] : column_list_) [[likely]] { if (name == col_name) { @@ -96,7 +99,7 @@ shift(size_type periods, shift_policy sp) const { static_assert(std::is_base_of, DataVec>::value, "Only a StdDataFrame can call shift()"); - DataFrame slug = *this; + DataFrame slug { *this }; slug.template self_shift(periods, sp); return (slug); @@ -113,7 +116,7 @@ shift(const char *col_name, size_type periods, shift_policy sp) const { "Only a StdDataFrame can call shift()"); ColumnVecType result = get_column(col_name); - vertical_shift_functor_ functor(periods, sp); + vertical_shift_functor_ functor { periods, sp }; functor (result); return (result); diff --git a/include/DataFrame/Internals/DataFrame_slice.tcc b/include/DataFrame/Internals/DataFrame_slice.tcc index 296a689f..72b60879 100644 --- a/include/DataFrame/Internals/DataFrame_slice.tcc +++ b/include/DataFrame/Internals/DataFrame_slice.tcc @@ -43,48 +43,60 @@ namespace hmdf template template DataFrame> -DataFrame::get_data_by_idx (Index2D range) const { +DataFrame::get_data_by_idx(Index2D range) const { using res_t = DataFrame>; - const auto &lower = - std::lower_bound (indices_.begin(), indices_.end(), range.begin); - const auto &upper = - std::upper_bound (indices_.begin(), indices_.end(), range.end); + const auto &lower { + std::lower_bound(indices_.begin(), indices_.end(), range.begin) + }; + const auto &upper { + std::upper_bound(indices_.begin(), indices_.end(), range.end) + }; res_t df; - if (lower != indices_.end()) { + // If range.begin > range.end, lower could be after upper. That is an empty + // range. + // + if (lower != indices_.end() && lower <= upper) { df.load_index(lower, upper); - const size_type b_dist = std::distance(indices_.begin(), lower); - const size_type e_dist = - std::distance(indices_.begin(), - upper < indices_.end() ? upper : indices_.end()); - const SpinGuard guard(lock_); + const size_type b_dist { + size_type(std::distance(indices_.begin(), lower)) + }; + const size_type e_dist { + size_type(std::distance(indices_.begin(), + upper < indices_.end() + ? upper : indices_.end())) + }; + const SpinGuard guard { lock_ }; for (const auto &[name, idx] : column_list_) [[likely]] { - create_col_functor_ functor(name.c_str(), df); + create_col_functor_ functor { name.c_str(), df }; data_[idx].change(functor); } - const auto thread_level = + const auto thread_level { (indices_.size() < ThreadPool::MUL_THR_THHOLD) - ? 0L : get_thread_level(); + ? 0L : get_thread_level() + }; auto lbd = [b_dist, e_dist, &df, this] (const auto &begin, const auto &end) -> void { - for (auto citer = begin; citer < end; ++citer) { - load_functor_ functor ( - citer->first.c_str(), b_dist, e_dist, df); + for (auto citer { begin }; citer < end; ++citer) { + load_functor_ functor { + citer->first.c_str(), b_dist, e_dist, df + }; this->data_[citer->second].change(functor); } }; if (thread_level > 2) { - auto futuers = thr_pool_.parallel_loop( - column_list_.begin(), column_list_.end(), std::move(lbd)); + auto futuers { thr_pool_.parallel_loop( + column_list_.begin(), column_list_.end(), std::move(lbd)) + }; for (auto &fut : futuers) fut.get(); } @@ -107,7 +119,7 @@ get_data_by_idx(const StlVecType &values) const { using res_t = DataFrame>; - const DFUnorderedSet val_table (values.begin(), values.end()); + const DFUnorderedSet val_table(values.begin(), values.end()); typename res_t::IndexVecType new_index; StlVecType locations; const size_type values_s = values.size(); @@ -178,33 +190,38 @@ get_data_by_idx(const StlVecType &values) const { template template typename DataFrame::View -DataFrame::get_view_by_idx (Index2D range) { - - const auto lower = - std::lower_bound (indices_.begin(), indices_.end(), range.begin); - const auto upper = - std::upper_bound (indices_.begin(), indices_.end(), range.end); +DataFrame::get_view_by_idx(Index2D range) { + + const auto lower { + std::lower_bound(indices_.begin(), indices_.end(), range.begin) + }; + const auto upper { + std::upper_bound(indices_.begin(), indices_.end(), range.end) + }; View dfv; - if (lower != indices_.end() && - (upper != indices_.end() || indices_.back() == range.end)) [[likely]] { - IndexType *upper_address { nullptr }; - const size_type b_dist = std::distance(indices_.begin(), lower); - const size_type e_dist = std::distance(indices_.begin(), upper); + // The range is inclusive. So, if range.end is at or beyond the last index, + // upper is the end iterator and the view extends to the last row, exactly + // the same as get_data_by_idx(). If range.begin > range.end, lower could be + // after upper. That is an empty range. + // + if (lower != indices_.end() && lower <= upper) [[likely]] { + const size_type b_dist { + size_type(std::distance(indices_.begin(), lower)) + }; + const size_type e_dist { + size_type(std::distance(indices_.begin(), upper)) + }; + IndexType *upper_address { &(indices_.front()) + e_dist }; - if (upper != indices_.end()) - upper_address = &*upper; - else - upper_address = &(indices_.front()) + e_dist; dfv.indices_ = typename View::IndexVecType(&*lower, upper_address); - const SpinGuard guard(lock_); + const SpinGuard guard { lock_ }; for (const auto &[name, idx] : column_list_) [[likely]] { - view_setup_functor_ functor (name.c_str(), - b_dist, - e_dist, - dfv); + view_setup_functor_ functor { + name.c_str(), b_dist, e_dist, dfv + }; data_[idx].change(functor); } @@ -218,34 +235,36 @@ DataFrame::get_view_by_idx (Index2D range) { template template typename DataFrame::ConstView -DataFrame::get_view_by_idx (Index2D range) const { - - const auto lower = - std::lower_bound (indices_.begin(), indices_.end(), range.begin); - const auto upper = - std::upper_bound (indices_.begin(), indices_.end(), range.end); +DataFrame::get_view_by_idx(Index2D range) const { + + const auto lower { + std::lower_bound (indices_.begin(), indices_.end(), range.begin) + }; + const auto upper { + std::upper_bound (indices_.begin(), indices_.end(), range.end) + }; ConstView dfcv; - if (lower != indices_.end() && - (upper != indices_.end() || indices_.back() == range.end)) [[likely]] { - const IndexType *upper_address = nullptr; - const size_type b_dist = std::distance(indices_.begin(), lower); - const size_type e_dist = std::distance(indices_.begin(), upper); + // See the non-const version + // + if (lower != indices_.end() && lower <= upper) [[likely]] { + const size_type b_dist { + size_type(std::distance(indices_.begin(), lower)) + }; + const size_type e_dist { + size_type(std::distance(indices_.begin(), upper)) + }; + const IndexType *upper_address { &(indices_.front()) + e_dist }; - if (upper != indices_.end()) - upper_address = &*upper; - else - upper_address = &(indices_.front()) + e_dist; dfcv.indices_ = typename ConstView::IndexVecType(&*lower, upper_address); - const SpinGuard guard(lock_); + const SpinGuard guard { lock_ }; for (const auto &[name, idx] : column_list_) [[likely]] { - view_setup_functor_ functor (name.c_str(), - b_dist, - e_dist, - dfcv); + view_setup_functor_ functor { + name.c_str(), b_dist, e_dist, dfcv + }; data_[idx].change(functor); } @@ -346,12 +365,16 @@ get_view_by_idx(const StlVecType &values) const { template template DataFrame> DataFrame:: -get_data_by_loc (Index2D range, inclusiveness incld) const { +get_data_by_loc(Index2D range, inclusiveness incld) const { using res_t = DataFrame>; if (range.begin < 0) range.begin = static_cast(indices_.size()) + range.begin; + // Python-like negative indexing: -1 is the last row, so, with the default + // inclusiveness, { x, -1 } does not include the last row. This is the same + // as remove_data_by_loc(). + // if (range.end < 0) range.end = static_cast(indices_.size()) + range.end + 1; @@ -365,51 +388,57 @@ get_data_by_loc (Index2D range, inclusiveness incld) const { df.load_index(indices_.begin() + col_begin, indices_.begin() + col_end); - const SpinGuard guard(lock_); + const SpinGuard guard { lock_ }; for (const auto &[name, idx] : column_list_) [[likely]] { - create_col_functor_ functor(name.c_str(), df); + create_col_functor_ functor { + name.c_str(), df + }; data_[idx].change(functor); } - const auto thread_level = + const auto thread_level { (indices_.size() < ThreadPool::MUL_THR_THHOLD) - ? 0L : get_thread_level(); + ? 0L : get_thread_level() + }; if (thread_level > 2) { auto lbd = [&range = std::as_const(range), &df, this, incld] - (const auto &begin, const auto &end) -> void { - for (auto citer = begin; citer < end; ++citer) { - load_functor_ functor( - citer->first.c_str(), - static_cast(range.begin), - static_cast(range.end), - df, - nan_policy::pad_with_nans, - incld); - - this->data_[citer->second].change(functor); - } - }; + (const auto &begin, const auto &end) -> void { + for (auto citer { begin }; citer < end; ++citer) { + load_functor_ functor { + citer->first.c_str(), + static_cast(range.begin), + static_cast(range.end), + df, + nan_policy::pad_with_nans, + incld + }; + + data_[citer->second].change(functor); + } + }; - auto futuers = + auto futuers { thr_pool_.parallel_loop(column_list_.begin(), column_list_.end(), - std::move(lbd)); + std::move(lbd)) + }; for (auto &fut : futuers) fut.get(); } else { for (const auto &[name, idx] : column_list_) [[likely]] { - load_functor_ functor( + load_functor_ functor { name.c_str(), static_cast(range.begin), static_cast(range.end), df, nan_policy::pad_with_nans, - incld); + incld + }; data_[idx].change(functor); } @@ -433,7 +462,7 @@ get_data_by_loc (Index2D range, inclusiveness incld) const { template template DataFrame> DataFrame:: -get_data_by_loc (const StlVecType &locations) const { +get_data_by_loc(const StlVecType &locations) const { using res_t = DataFrame>; @@ -502,9 +531,9 @@ get_data_by_loc (const StlVecType &locations) const { template template typename DataFrame::View DataFrame:: -get_view_by_loc (Index2D range, inclusiveness incld) { +get_view_by_loc(Index2D range, inclusiveness incld) { - const long idx_s = static_cast(indices_.size()); + const long idx_s { static_cast(indices_.size()) }; if (range.begin < 0) range.begin = idx_s + range.begin; @@ -522,15 +551,16 @@ get_view_by_loc (Index2D range, inclusiveness incld) { typename View::IndexVecType(&(indices_[0]) + col_begin, &(indices_[0]) + col_end); - const SpinGuard guard(lock_); + const SpinGuard guard { lock_ }; for (const auto &[name, idx] : column_list_) [[likely]] { - view_setup_functor_ functor ( + view_setup_functor_ functor { name.c_str(), static_cast(range.begin), static_cast(range.end), dfv, - incld); + incld + }; data_[idx].change(functor); } @@ -552,9 +582,9 @@ get_view_by_loc (Index2D range, inclusiveness incld) { template template typename DataFrame::ConstView DataFrame:: -get_view_by_loc (Index2D range, inclusiveness incld) const { +get_view_by_loc(Index2D range, inclusiveness incld) const { - const long idx_s = static_cast(indices_.size()); + const long idx_s { static_cast(indices_.size()) }; if (range.begin < 0) range.begin = idx_s + range.begin; @@ -572,15 +602,16 @@ get_view_by_loc (Index2D range, inclusiveness incld) const { typename ConstView::IndexVecType(&(indices_[0]) + col_begin, &(indices_[0]) + col_end); - const SpinGuard guard(lock_); + const SpinGuard guard { lock_ }; for (const auto &[name, idx] : column_list_) [[likely]] { - view_setup_functor_ functor ( + view_setup_functor_ functor { name.c_str(), static_cast(range.begin), static_cast(range.end), dfcv, - incld); + incld + }; data_[idx].change(functor); } @@ -603,7 +634,7 @@ get_view_by_loc (Index2D range, inclusiveness incld) const { template template typename DataFrame::PtrView -DataFrame::get_view_by_loc (const StlVecType &locations) { +DataFrame::get_view_by_loc(const StlVecType &locations) { using TheView = PtrView; @@ -642,7 +673,7 @@ template template typename DataFrame::ConstPtrView DataFrame:: -get_view_by_loc (const StlVecType &locations) const { +get_view_by_loc(const StlVecType &locations) const { using TheView = ConstPtrView; @@ -680,7 +711,7 @@ get_view_by_loc (const StlVecType &locations) const { template template DataFrame> DataFrame:: -get_data_by_sel (const char *name, F &sel_functor) const requires +get_data_by_sel(const char *name, F &sel_functor) const requires std::invocable && std::same_as, bool> { @@ -702,7 +733,7 @@ std::same_as, bool> { template template typename DataFrame::PtrView DataFrame:: -get_view_by_sel (const char *name, F &sel_functor) requires +get_view_by_sel(const char *name, F &sel_functor) requires std::invocable && std::same_as, bool> { @@ -724,7 +755,7 @@ std::same_as, bool> { template template typename DataFrame::ConstPtrView DataFrame:: -get_view_by_sel (const char *name, F &sel_functor) const requires +get_view_by_sel(const char *name, F &sel_functor) const requires std::invocable && std::same_as, bool> { @@ -746,8 +777,8 @@ std::same_as, bool> { template template DataFrame> DataFrame:: -get_data_by_sel (const char *name1, const char *name2, - F &sel_functor) const requires +get_data_by_sel(const char *name1, const char *name2, + F &sel_functor) const requires std::invocable && std::same_as, bool> { @@ -779,8 +810,8 @@ std::same_as template typename DataFrame::PtrView DataFrame:: -get_view_by_sel (const char *name1, const char *name2, - F &sel_functor) requires +get_view_by_sel(const char *name1, const char *name2, + F &sel_functor) requires std::invocable && std::same_as, bool> { @@ -813,8 +844,8 @@ template template typename DataFrame::ConstPtrView DataFrame:: -get_view_by_sel (const char *name1, const char *name2, - F &sel_functor) const requires +get_view_by_sel(const char *name1, const char *name2, + F &sel_functor) const requires std::invocable && std::same_as, bool> { @@ -846,10 +877,10 @@ std::same_as template DataFrame> DataFrame:: -get_data_by_sel (const char *name1, - const char *name2, - const char *name3, - F &sel_functor) const requires +get_data_by_sel(const char *name1, + const char *name2, + const char *name3, + F &sel_functor) const requires std::invocable && std::same_as template DataFrame DataFrame:: -get_data_by_sel (F &sel_functor) const { +get_data_by_sel(F &sel_functor) const { const size_type idx_s = indices_.size(); // Get columns to std::tuple @@ -981,7 +1012,7 @@ get_data_by_sel (F &sel_functor) const { template template DataFrame DataFrame:: -get_data_by_sel (F &sel_functor, FilterCols && ... filter_cols) const { +get_data_by_sel(F &sel_functor, FilterCols && ... filter_cols) const { const size_type idx_s = indices_.size(); // Get columns to std::tuple @@ -1076,10 +1107,10 @@ get_data_by_sel (F &sel_functor, FilterCols && ... filter_cols) const { template template typename DataFrame::PtrView DataFrame:: -get_view_by_sel (const char *name1, - const char *name2, - const char *name3, - F &sel_functor) requires +get_view_by_sel(const char *name1, + const char *name2, + const char *name3, + F &sel_functor) requires std::invocable && std::same_as template typename DataFrame::ConstPtrView DataFrame:: -get_view_by_sel (const char *name1, - const char *name2, - const char *name3, - F &sel_functor) const requires +get_view_by_sel(const char *name1, + const char *name2, + const char *name3, + F &sel_functor) const requires std::invocable && std::same_as template typename DataFrame::PtrView DataFrame:: -get_view_by_rand (random_policy spec, double n, seed_t seed) { +get_view_by_rand(random_policy spec, double n, seed_t seed) { bool use_seed = false; size_type n_rows = static_cast(n); @@ -2144,7 +2175,7 @@ template template typename DataFrame::ConstPtrView DataFrame:: -get_view_by_rand (random_policy spec, double n, seed_t seed) const { +get_view_by_rand(random_policy spec, double n, seed_t seed) const { bool use_seed = false; size_type n_rows = static_cast(n); @@ -2220,12 +2251,13 @@ template DataFrame DataFrame:: get_data_every_n(size_type n, size_type starting_idx) const { - const size_type idx_s = indices_.size(); + const size_type idx_s { indices_.size() }; #ifdef HMDF_SANITY_EXCEPTIONS - if (n >= (idx_s - 1) || starting_idx >= idx_s) + if (n > (idx_s - 1) || starting_idx >= idx_s || n == 0) throw NotFeasible("get_data_every_n(): " - "n and starting_idx must be < index column length"); + "n and starting_idx must be < index column length " + "and n > 0"); #endif // HMDF_SANITY_EXCEPTIONS StlVecType col_indices; @@ -2244,12 +2276,13 @@ template typename DataFrame::PtrView DataFrame:: get_view_every_n(size_type n, size_type starting_idx) { - const size_type idx_s = indices_.size(); + const size_type idx_s { indices_.size() }; #ifdef HMDF_SANITY_EXCEPTIONS - if (n >= (idx_s - 1) || starting_idx >= idx_s) - throw NotFeasible("get_data_every_n(): " - "n and starting_idx must be < index column length"); + if (n > (idx_s - 1) || starting_idx >= idx_s || n == 0) + throw NotFeasible("get_view_every_n(): " + "n and starting_idx must be < index column length " + "and n > 0"); #endif // HMDF_SANITY_EXCEPTIONS StlVecType col_indices; @@ -2268,12 +2301,13 @@ template typename DataFrame::ConstPtrView DataFrame:: get_view_every_n(size_type n, size_type starting_idx) const { - const size_type idx_s = indices_.size(); + const size_type idx_s { indices_.size() }; #ifdef HMDF_SANITY_EXCEPTIONS - if (n >= (idx_s - 1) || starting_idx >= idx_s) - throw NotFeasible("get_data_every_n(): " - "n and starting_idx must be < index column length"); + if (n > (idx_s - 1) || starting_idx >= idx_s || n == 0) + throw NotFeasible("get_view_every_n(): " + "n and starting_idx must be < index column length " + "and n >0"); #endif // HMDF_SANITY_EXCEPTIONS StlVecType col_indices; @@ -2603,7 +2637,7 @@ template DataFrame DataFrame:: get_n_largest_data(const char *col_name, size_type n, bool abs_value) const { - const size_type idx_s = indices_.size(); + const size_type idx_s { indices_.size() }; #ifdef HMDF_SANITY_EXCEPTIONS if (n >= idx_s) @@ -2611,15 +2645,12 @@ get_n_largest_data(const char *col_name, size_type n, bool abs_value) const { "n must be < index column length"); #endif // HMDF_SANITY_EXCEPTIONS - const auto per_vec = - permutation_vec( + const StlVecType col_indices { + n_permutation_vec_( col_name, - abs_value ? sort_spec::abs_desce : sort_spec::desce); - const auto col_s = std::min(n, per_vec.size()); - StlVecType col_indices(col_s); - - for (size_type i = 0; i < col_s; ++i) - col_indices[i] = per_vec[i]; + abs_value ? sort_spec::abs_desce : sort_spec::desce, + n) + }; return (data_by_sel_common_(col_indices, idx_s)); } @@ -2639,15 +2670,12 @@ get_n_largest_view(const char *col_name, size_type n, bool abs_value) { "n must be < index column length"); #endif // HMDF_SANITY_EXCEPTIONS - const auto per_vec = - permutation_vec( + const StlVecType col_indices { + n_permutation_vec_( col_name, - abs_value ? sort_spec::abs_desce : sort_spec::desce); - const auto col_s = std::min(n, per_vec.size()); - StlVecType col_indices(col_s); - - for (size_type i = 0; i < col_s; ++i) - col_indices[i] = per_vec[i]; + abs_value ? sort_spec::abs_desce : sort_spec::desce, + n) + }; return (view_by_sel_common_(col_indices, idx_s)); } @@ -2667,15 +2695,12 @@ get_n_largest_view(const char *col_name, size_type n, bool abs_value) const { "n must be < index column length"); #endif // HMDF_SANITY_EXCEPTIONS - const auto per_vec = - permutation_vec( + const StlVecType col_indices { + n_permutation_vec_( col_name, - abs_value ? sort_spec::abs_desce : sort_spec::desce); - const auto col_s = std::min(n, per_vec.size()); - StlVecType col_indices(col_s); - - for (size_type i = 0; i < col_s; ++i) - col_indices[i] = per_vec[i]; + abs_value ? sort_spec::abs_desce : sort_spec::desce, + n) + }; return (view_by_sel_common_(col_indices, idx_s)); } @@ -2695,15 +2720,12 @@ get_n_smallest_data(const char *col_name, size_type n, bool abs_value) const { "n must be < index column length"); #endif // HMDF_SANITY_EXCEPTIONS - const auto per_vec = - permutation_vec( + const StlVecType col_indices { + n_permutation_vec_( col_name, - abs_value ? sort_spec::abs_ascen : sort_spec::ascen); - const auto col_s = std::min(n, per_vec.size()); - StlVecType col_indices(col_s); - - for (size_type i = 0; i < col_s; ++i) - col_indices[i] = per_vec[i]; + abs_value ? sort_spec::abs_ascen : sort_spec::ascen, + n) + }; return (data_by_sel_common_(col_indices, idx_s)); } @@ -2723,15 +2745,12 @@ get_n_smallest_view(const char *col_name, size_type n, bool abs_value) { "n must be < index column length"); #endif // HMDF_SANITY_EXCEPTIONS - const auto per_vec = - permutation_vec( + const StlVecType col_indices { + n_permutation_vec_( col_name, - abs_value ? sort_spec::abs_ascen : sort_spec::ascen); - const auto col_s = std::min(n, per_vec.size()); - StlVecType col_indices(col_s); - - for (size_type i = 0; i < col_s; ++i) - col_indices[i] = per_vec[i]; + abs_value ? sort_spec::abs_ascen : sort_spec::ascen, + n) + }; return (view_by_sel_common_(col_indices, idx_s)); } @@ -2751,15 +2770,12 @@ get_n_smallest_view(const char *col_name, size_type n, bool abs_value) const { "n must be < index column length"); #endif // HMDF_SANITY_EXCEPTIONS - const auto per_vec = - permutation_vec( + const StlVecType col_indices { + n_permutation_vec_( col_name, - abs_value ? sort_spec::abs_ascen : sort_spec::ascen); - const auto col_s = std::min(n, per_vec.size()); - StlVecType col_indices(col_s); - - for (size_type i = 0; i < col_s; ++i) - col_indices[i] = per_vec[i]; + abs_value ? sort_spec::abs_ascen : sort_spec::ascen, + n) + }; return (view_by_sel_common_(col_indices, idx_s)); } @@ -2777,8 +2793,8 @@ get_data_by_stdev(const char *col_name, T high_stdev, T low_stdev) const { const auto thread_level = (col_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level(); auto mean_lbd = - [&vec = std::as_const(vec), this]() -> T { - MeanVisitor mean { true }; + [&vec = std::as_const(vec), this]() -> double { + MeanVisitor mean { true }; mean.pre(); mean(indices_.begin(), indices_.end(), vec.begin(), vec.end()); @@ -2786,16 +2802,16 @@ get_data_by_stdev(const char *col_name, T high_stdev, T low_stdev) const { return (mean.get_result()); }; auto stdev_lbd = - [&vec = std::as_const(vec), this]() -> T { - StdVisitor stdev { true, true }; + [&vec = std::as_const(vec), this]() -> double { + StdVisitor stdev { true, true }; stdev.pre(); stdev(indices_.begin(), indices_.end(), vec.begin(), vec.end()); stdev.post(); return (stdev.get_result()); }; - T stdev; - T mean; + double stdev; + double mean; if (thread_level > 2) { auto stdev_fut = thr_pool_.dispatch(false, stdev_lbd); @@ -2813,7 +2829,7 @@ get_data_by_stdev(const char *col_name, T high_stdev, T low_stdev) const { col_indices.reserve(col_s / 2); for (size_type i = 0; i < col_s; ++i) { - const T z = (vec[i] - mean) / stdev; + const double z = (double(vec[i]) - mean) / stdev; if (z < high_stdev && z > low_stdev) col_indices.push_back(i); @@ -2835,8 +2851,8 @@ get_view_by_stdev(const char *col_name, T high_stdev, T low_stdev) { const auto thread_level = (col_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level(); auto mean_lbd = - [&vec = std::as_const(vec), this]() -> T { - MeanVisitor mean { true }; + [&vec = std::as_const(vec), this]() -> double { + MeanVisitor mean { true }; mean.pre(); mean(indices_.begin(), indices_.end(), vec.begin(), vec.end()); @@ -2844,16 +2860,16 @@ get_view_by_stdev(const char *col_name, T high_stdev, T low_stdev) { return (mean.get_result()); }; auto stdev_lbd = - [&vec = std::as_const(vec), this]() -> T { - StdVisitor stdev { true, true }; + [&vec = std::as_const(vec), this]() -> double { + StdVisitor stdev { true, true }; stdev.pre(); stdev(indices_.begin(), indices_.end(), vec.begin(), vec.end()); stdev.post(); return (stdev.get_result()); }; - T stdev; - T mean; + double stdev; + double mean; if (thread_level > 2) { auto stdev_fut = thr_pool_.dispatch(false, stdev_lbd); @@ -2871,7 +2887,7 @@ get_view_by_stdev(const char *col_name, T high_stdev, T low_stdev) { col_indices.reserve(col_s / 2); for (size_type i = 0; i < col_s; ++i) { - const T z = (vec[i] - mean) / stdev; + const double z = (double(vec[i]) - mean) / stdev; if (z < high_stdev && z > low_stdev) col_indices.push_back(i); @@ -2893,8 +2909,8 @@ get_view_by_stdev(const char *col_name, T high_stdev, T low_stdev) const { const auto thread_level = (col_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level(); auto mean_lbd = - [&vec = std::as_const(vec), this]() -> T { - MeanVisitor mean { true }; + [&vec = std::as_const(vec), this]() -> double { + MeanVisitor mean { true }; mean.pre(); mean(indices_.begin(), indices_.end(), vec.begin(), vec.end()); @@ -2902,16 +2918,16 @@ get_view_by_stdev(const char *col_name, T high_stdev, T low_stdev) const { return (mean.get_result()); }; auto stdev_lbd = - [&vec = std::as_const(vec), this]() -> T { - StdVisitor stdev { true, true }; + [&vec = std::as_const(vec), this]() -> double { + StdVisitor stdev { true, true }; stdev.pre(); stdev(indices_.begin(), indices_.end(), vec.begin(), vec.end()); stdev.post(); return (stdev.get_result()); }; - T stdev; - T mean; + double stdev; + double mean; if (thread_level > 2) { auto stdev_fut = thr_pool_.dispatch(false, stdev_lbd); @@ -2929,7 +2945,7 @@ get_view_by_stdev(const char *col_name, T high_stdev, T low_stdev) const { col_indices.reserve(col_s / 2); for (size_type i = 0; i < col_s; ++i) { - const T z = (vec[i] - mean) / stdev; + const double z = (double(vec[i]) - mean) / stdev; if (z < high_stdev && z > low_stdev) col_indices.push_back(i); @@ -3622,7 +3638,24 @@ get_view_at_times(DateTime::HourType hr, // ---------------------------------------------------------------------------- +// Time of day as a single comparable number, so that comparing two times is +// a plain lexicographic (hour, minute, second, millisecond) comparison. +// +inline static std::int64_t +_DT_time_key_(DateTime::HourType hr, + DateTime::MinuteType mn, + DateTime::SecondType sc, + DateTime::MillisecondType msc) { + + return (((std::int64_t(hr) * 60 + std::int64_t(mn)) * 60 + + std::int64_t(sc)) * 1000 + std::int64_t(msc)); +} + +// ---------------------------------------------------------------------------- + // Benchmark time vs. Data time +// Returns true if the data time is before (or equal to, if incld is true) the +// benchmark time. // inline static bool _DT_before_times_(DateTime::HourType b_hr, @@ -3635,17 +3668,17 @@ _DT_before_times_(DateTime::HourType b_hr, DateTime::MillisecondType d_msc, bool incld = false) { - if (! incld) [[likely]] - return ((d_hr < b_hr) || - (d_hr == b_hr && d_mn < b_mn) || - (d_mn == b_mn && d_sc < b_sc) || - (d_sc == b_sc && d_msc < b_msc)); - return (d_hr <= b_hr && d_mn <= b_mn && d_sc <= b_sc && d_msc <= b_msc); + const std::int64_t b { _DT_time_key_(b_hr, b_mn, b_sc, b_msc) }; + const std::int64_t d { _DT_time_key_(d_hr, d_mn, d_sc, d_msc) }; + + return (incld ? d <= b : d < b); } // ---------------------------------------------------------------------------- // Benchmark time vs. Data time +// Returns true if the data time is after (or equal to, if incld is true) the +// benchmark time. // inline static bool _DT_after_times_(DateTime::HourType b_hr, @@ -3658,12 +3691,10 @@ _DT_after_times_(DateTime::HourType b_hr, DateTime::MillisecondType d_msc, bool incld = false) { - if (! incld) [[likely]] - return ((d_hr > b_hr) || - (d_hr == b_hr && d_mn > b_mn) || - (d_mn == b_mn && d_sc > b_sc) || - (d_sc == b_sc && d_msc > b_msc)); - return (d_hr >= b_hr && d_mn >= b_mn && d_sc >= b_sc && d_msc >= b_msc); + const std::int64_t b { _DT_time_key_(b_hr, b_mn, b_sc, b_msc) }; + const std::int64_t d { _DT_time_key_(d_hr, d_mn, d_sc, d_msc) }; + + return (incld ? d >= b : d > b); } // ---------------------------------------------------------------------------- diff --git a/include/DataFrame/Internals/DataFrame_sort.tcc b/include/DataFrame/Internals/DataFrame_sort.tcc index 33f90d64..3e246e5a 100644 --- a/include/DataFrame/Internals/DataFrame_sort.tcc +++ b/include/DataFrame/Internals/DataFrame_sort.tcc @@ -43,7 +43,7 @@ permutation_vec(const char *name, sort_spec dir) const { const ColumnVecType *vec { nullptr }; { - const SpinGuard guard (lock_); + const SpinGuard guard { lock_ }; if (! ::strcmp(name, DF_INDEX_COL_NAME)) vec = reinterpret_cast *>(&indices_); @@ -51,10 +51,11 @@ permutation_vec(const char *name, sort_spec dir) const { vec = &(get_column(name, false)); } - const size_type col_s = vec->size(); + const size_type col_s { vec->size() }; std::vector result(col_s); - const auto thread_level = - (col_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level(); + const auto thread_level { + (col_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level() + }; std::iota(result.begin(), result.end(), 0); if (dir == sort_spec::ascen) { @@ -122,7 +123,7 @@ sort(const char *name, sort_spec dir, bool ignore_index) { make_consistent(); ColumnVecType *vec { nullptr }; - const SpinGuard guard (lock_); + const SpinGuard guard { lock_ }; if (! ::strcmp(name, DF_INDEX_COL_NAME)) { vec = reinterpret_cast *>(&indices_); @@ -144,16 +145,18 @@ sort(const char *name, sort_spec dir, bool ignore_index) { return (abs__(std::get<0>(lhs)) > abs__(std::get<0>(rhs))); }; - const size_type idx_s = indices_.size(); + const size_type idx_s { indices_.size() }; StlVecType sorting_idxs(idx_s); std::iota(sorting_idxs.begin(), sorting_idxs.end(), 0); - auto zip = std::ranges::views::zip(*vec, sorting_idxs); - auto zip_idx = - std::ranges::views::zip(*vec, indices_, sorting_idxs); - const auto thread_level = - (idx_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level(); + auto zip { std::ranges::views::zip(*vec, sorting_idxs) }; + auto zip_idx { + std::ranges::views::zip(*vec, indices_, sorting_idxs) + }; + const auto thread_level { + (idx_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level() + }; if (dir == sort_spec::ascen) { if (thread_level > 2) { @@ -223,9 +226,10 @@ sort(const char *name, sort_spec dir, bool ignore_index) { if (citer->first != name) this->data_[citer->second].change(functor); }; - auto futures = + auto futures { thr_pool_.parallel_loop( - column_list_.begin(), column_list_.end(), std::move(lbd)); + column_list_.begin(), column_list_.end(), std::move(lbd)) + }; for (auto &fut : futures) fut.get(); } @@ -254,7 +258,7 @@ sort(const char *name1, sort_spec dir1, ColumnVecType *vec1 { nullptr }; ColumnVecType *vec2 { nullptr }; - const SpinGuard guard (lock_); + const SpinGuard guard { lock_ }; if (! ::strcmp(name1, DF_INDEX_COL_NAME)) { vec1 = reinterpret_cast *>(&indices_); @@ -367,16 +371,18 @@ sort(const char *name1, sort_spec dir1, return (abs__(std::get<0>(lhs)) > abs__(std::get<0>(rhs))); }; - const size_type idx_s = indices_.size(); + const size_type idx_s { indices_.size() }; StlVecType sorting_idxs(idx_s); std::iota(sorting_idxs.begin(), sorting_idxs.end(), 0); - auto zip = std::ranges::views::zip(*vec1, *vec2, sorting_idxs); - auto zip_idx = - std::ranges::views::zip(*vec1, *vec2, indices_, sorting_idxs); - const auto thread_level = - (idx_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level(); + auto zip { std::ranges::views::zip(*vec1, *vec2, sorting_idxs) }; + auto zip_idx { + std::ranges::views::zip(*vec1, *vec2, indices_, sorting_idxs) + }; + const auto thread_level { + (idx_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level() + }; if (dir1 == sort_spec::ascen && dir2 == sort_spec::ascen) { if (thread_level > 2) { @@ -614,9 +620,11 @@ sort(const char *name1, sort_spec dir1, if (citer->first != name1 && citer->first != name2) this->data_[citer->second].change(functor); }; - auto futures = + auto futures { thr_pool_.parallel_loop( - column_list_.begin(), column_list_.end(), std::move(lbd)); + column_list_.begin(), column_list_.end(), std::move(lbd)) + + }; for (auto &fut : futures) fut.get(); } @@ -643,7 +651,7 @@ permutation_vec(const char *name1, sort_spec dir1, const ColumnVecType *vec2 { nullptr }; { - const SpinGuard guard (lock_); + const SpinGuard guard { lock_ }; if (! ::strcmp(name1, DF_INDEX_COL_NAME)) vec1 = reinterpret_cast *>(&indices_); @@ -656,10 +664,11 @@ permutation_vec(const char *name1, sort_spec dir1, vec2 = &(get_column(name2, false)); } - const size_type col_s = std::min(vec1->size(), vec2->size()); + const size_type col_s { std::min(vec1->size(), vec2->size()) }; std::vector result(col_s); - const auto thread_level = - (col_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level(); + const auto thread_level { + (col_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level() + }; std::iota(result.begin(), result.end(), 0); if (dir1 == sort_spec::ascen && dir2 == sort_spec::ascen) { @@ -908,7 +917,7 @@ sort(const char *name1, sort_spec dir1, ColumnVecType *vec1 { nullptr }; ColumnVecType *vec2 { nullptr }; ColumnVecType *vec3 { nullptr }; - const SpinGuard guard (lock_); + const SpinGuard guard { lock_ }; if (! ::strcmp(name1, DF_INDEX_COL_NAME)) { vec1 = reinterpret_cast *>(&indices_); @@ -993,17 +1002,20 @@ sort(const char *name1, sort_spec dir1, return (abs__(std::get<2>(lhs)) > abs__(std::get<2>(rhs))); }; - const size_type idx_s = indices_.size(); + const size_type idx_s { indices_.size() }; StlVecType sorting_idxs(idx_s); std::iota(sorting_idxs.begin(), sorting_idxs.end(), 0); - auto zip = - std::ranges::views::zip(*vec1, *vec2, *vec3, sorting_idxs); - auto zip_idx = - std::ranges::views::zip(*vec1, *vec2, *vec3, indices_, sorting_idxs); - const auto thread_level = - (idx_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level(); + auto zip { + std::ranges::views::zip(*vec1, *vec2, *vec3, sorting_idxs) + }; + auto zip_idx { + std::ranges::views::zip(*vec1, *vec2, *vec3, indices_, sorting_idxs) + }; + const auto thread_level { + (idx_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level() + }; if (thread_level > 2) { if (! ignore_index) @@ -1012,10 +1024,8 @@ sort(const char *name1, sort_spec dir1, thr_pool_.parallel_sort(zip.begin(), zip.end(), cf); } else { - if (! ignore_index) - std::ranges::sort(zip_idx, cf); - else - std::ranges::sort(zip, cf); + if (! ignore_index) std::ranges::sort(zip_idx, cf); + else std::ranges::sort(zip, cf); } if (((column_list_.size() - 3) > 1) && get_thread_level() > 2) { @@ -1031,9 +1041,10 @@ sort(const char *name1, sort_spec dir1, citer->first != name3) this->data_[citer->second].change(functor); }; - auto futures = + auto futures { thr_pool_.parallel_loop( - column_list_.begin(), column_list_.end(), std::move(lbd)); + column_list_.begin(), column_list_.end(), std::move(lbd)) + }; for (auto &fut : futures) fut.get(); } @@ -1064,7 +1075,7 @@ permutation_vec(const char *name1, sort_spec dir1, const ColumnVecType *vec3 { nullptr }; { - const SpinGuard guard (lock_); + const SpinGuard guard { lock_ }; if (! ::strcmp(name1, DF_INDEX_COL_NAME)) vec1 = reinterpret_cast *>(&indices_); @@ -1148,11 +1159,13 @@ permutation_vec(const char *name1, sort_spec dir1, return (abs__(vec3[lhs]) > abs__(vec3[rhs])); }; - const size_type col_s = - std::min({ vec1->size(), vec2->size(), vec3->size() }); + const size_type col_s { + std::min({ vec1->size(), vec2->size(), vec3->size() }) + }; std::vector result(col_s); - const auto thread_level = - (col_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level(); + const auto thread_level { + (col_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level() + }; std::iota(result.begin(), result.end(), 0); if (thread_level > 2) @@ -1179,11 +1192,11 @@ sort(const char *name1, sort_spec dir1, make_consistent(); - const ColumnVecType *vec1 { nullptr }; - const ColumnVecType *vec2 { nullptr }; - const ColumnVecType *vec3 { nullptr }; - const ColumnVecType *vec4 { nullptr }; - const SpinGuard guard (lock_); + ColumnVecType *vec1 { nullptr }; + ColumnVecType *vec2 { nullptr }; + ColumnVecType *vec3 { nullptr }; + ColumnVecType *vec4 { nullptr }; + const SpinGuard guard { lock_ }; if (! ::strcmp(name1, DF_INDEX_COL_NAME)) { vec1 = reinterpret_cast *>(&indices_); @@ -1300,18 +1313,21 @@ sort(const char *name1, sort_spec dir1, return (abs__(std::get<3>(lhs)) > abs__(std::get<3>(rhs))); }; - const size_type idx_s = indices_.size(); + const size_type idx_s { indices_.size() }; StlVecType sorting_idxs(idx_s); std::iota(sorting_idxs.begin(), sorting_idxs.end(), 0); - auto zip = - std::ranges::views::zip(*vec1, *vec2, *vec3, *vec4, sorting_idxs); - auto zip_idx = + auto zip { + std::ranges::views::zip(*vec1, *vec2, *vec3, *vec4, sorting_idxs) + }; + auto zip_idx { std::ranges::views::zip(*vec1, *vec2, *vec3, *vec4, - indices_, sorting_idxs); - const auto thread_level = - (idx_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level(); + indices_, sorting_idxs) + }; + const auto thread_level { + (idx_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level() + }; if (thread_level > 2) { if (! ignore_index) @@ -1320,10 +1336,8 @@ sort(const char *name1, sort_spec dir1, thr_pool_.parallel_sort(zip.begin(), zip.end(), cf); } else { - if (! ignore_index) - std::ranges::sort(zip_idx, cf); - else - std::ranges::sort(zip, cf); + if (! ignore_index) std::ranges::sort(zip_idx, cf); + else std::ranges::sort(zip, cf); } if (((column_list_.size() - 4) > 1) && get_thread_level() > 2) { @@ -1340,9 +1354,10 @@ sort(const char *name1, sort_spec dir1, citer->first != name4) this->data_[citer->second].change(functor); }; - auto futures = + auto futures { thr_pool_.parallel_loop( - column_list_.begin(), column_list_.end(), std::move(lbd)); + column_list_.begin(), column_list_.end(), std::move(lbd)) + }; for (auto &fut : futures) fut.get(); } @@ -1377,12 +1392,12 @@ sort(const char *name1, sort_spec dir1, make_consistent(); - const ColumnVecType *vec1 { nullptr }; - const ColumnVecType *vec2 { nullptr }; - const ColumnVecType *vec3 { nullptr }; - const ColumnVecType *vec4 { nullptr }; - const ColumnVecType *vec5 { nullptr }; - const SpinGuard guard (lock_); + ColumnVecType *vec1 { nullptr }; + ColumnVecType *vec2 { nullptr }; + ColumnVecType *vec3 { nullptr }; + ColumnVecType *vec4 { nullptr }; + ColumnVecType *vec5 { nullptr }; + const SpinGuard guard { lock_ }; if (! ::strcmp(name1, DF_INDEX_COL_NAME)) { vec1 = reinterpret_cast *>(&indices_); @@ -1412,7 +1427,7 @@ sort(const char *name1, sort_spec dir1, else vec4 = &(get_column(name4, false)); - if (! ::strcmp(name4, DF_INDEX_COL_NAME)) { + if (! ::strcmp(name5, DF_INDEX_COL_NAME)) { vec5 = reinterpret_cast *>(&indices_); ignore_index = true; } @@ -1532,19 +1547,22 @@ sort(const char *name1, sort_spec dir1, return (abs__(std::get<4>(lhs)) > abs__(std::get<4>(rhs))); }; - const size_type idx_s = indices_.size(); + const size_type idx_s { indices_.size() }; StlVecType sorting_idxs(idx_s); std::iota(sorting_idxs.begin(), sorting_idxs.end(), 0); - auto zip = + auto zip { std::ranges::views::zip(*vec1, *vec2, *vec3, *vec4, *vec5, - sorting_idxs); - auto zip_idx = + sorting_idxs) + }; + auto zip_idx { std::ranges::views::zip(*vec1, *vec2, *vec3, *vec4, *vec5, - indices_, sorting_idxs); - const auto thread_level = - (idx_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level(); + indices_, sorting_idxs) + }; + const auto thread_level { + (idx_s < ThreadPool::MUL_THR_THHOLD) ? 0L : get_thread_level() + }; if (thread_level > 2) { if (! ignore_index) @@ -1553,10 +1571,8 @@ sort(const char *name1, sort_spec dir1, thr_pool_.parallel_sort(zip.begin(), zip.end(), cf); } else { - if (! ignore_index) - std::ranges::sort(zip_idx, cf); - else - std::ranges::sort(zip, cf); + if (! ignore_index) std::ranges::sort(zip_idx, cf); + else std::ranges::sort(zip, cf); } if (((column_list_.size() - 5) > 1) && get_thread_level() > 2) { @@ -1574,9 +1590,10 @@ sort(const char *name1, sort_spec dir1, citer->first != name5) this->data_[citer->second].change(functor); }; - auto futures = + auto futures { thr_pool_.parallel_loop( - column_list_.begin(), column_list_.end(), std::move(lbd)); + column_list_.begin(), column_list_.end(), std::move(lbd)) + }; for (auto &fut : futures) fut.get(); } @@ -1725,7 +1742,7 @@ sort_freq(const char *name, sort_spec dir, bool ignore_index) { else vec = &(get_column(name)); - const size_type idx_s = indices_.size(); + const size_type idx_s { indices_.size() }; DFUnorderedMap freq_map; if (dir == sort_spec::ascen || dir == sort_spec::desce) { @@ -1792,11 +1809,13 @@ sort_freq(const char *name, sort_spec dir, bool ignore_index) { std::iota(sorting_idxs.begin(), sorting_idxs.end(), 0); - auto zip = std::ranges::views::zip(*vec, sorting_idxs); - auto zip_idx = - std::ranges::views::zip(*vec, indices_, sorting_idxs); - const auto thread_level = - (idx_s < (ThreadPool::MUL_THR_THHOLD / 3)) ? 0L : get_thread_level(); + auto zip { std::ranges::views::zip(*vec, sorting_idxs) }; + auto zip_idx { + std::ranges::views::zip(*vec, indices_, sorting_idxs) + }; + const auto thread_level { + (idx_s < (ThreadPool::MUL_THR_THHOLD / 3)) ? 0L : get_thread_level() + }; if (dir == sort_spec::ascen) { if (thread_level > 2) { @@ -1848,10 +1867,8 @@ sort_freq(const char *name, sort_spec dir, bool ignore_index) { thr_pool_.parallel_sort(zip.begin(), zip.end(), ad); } else { - if (! ignore_index) - std::ranges::sort(zip_idx, ad); - else - std::ranges::sort(zip, ad); + if (! ignore_index) std::ranges::sort(zip_idx, ad); + else std::ranges::sort(zip, ad); } } diff --git a/include/DataFrame/Internals/DataFrame_standalone.tcc b/include/DataFrame/Internals/DataFrame_standalone.tcc index c1ea60cb..413bcae2 100644 --- a/include/DataFrame/Internals/DataFrame_standalone.tcc +++ b/include/DataFrame/Internals/DataFrame_standalone.tcc @@ -2859,6 +2859,20 @@ struct _LikeClauseUtil_ { // ---------------------------------------------------------------------------- +// NO_WILDCARD_MATCH means that there is no match, and that the input was +// exhausted while the rest of the pattern was still looking for something +// after a '*'. So, no later starting position of that '*' can match +// either, and the caller's '*' should stop retrying. Without this, a +// pattern with several '*' takes exponential time on an input that does +// not match. +// +enum class like_clause_res : unsigned char { + + NO_MATCH = 0, + MATCH = 1, + NO_MATCH_INPUT_EXHAUSTED = 2, +}; + // This compares two null-terminated strings for equality where the first // string can potentially be a "glob" expression (forget about regular // expressions). @@ -2889,18 +2903,20 @@ struct _LikeClauseUtil_ { // moderately sized strings. I have not tested this with huge/massive // strings. // -static inline bool -_like_clause_compare_(const char *pattern, - const char *input_str, - bool case_insensitive = false, - unsigned int esc_char = '\\') { +static inline like_clause_res +_like_clause_compare_impl_(const char *pattern, + const char *input_str, + bool case_insensitive, + unsigned int esc_char) { using value_type = _LikeClauseUtil_::value_type; - const value_type *upattern = - reinterpret_cast(pattern); - const value_type *uinput_str = - reinterpret_cast(input_str); + const value_type *upattern { + reinterpret_cast(pattern) + }; + const value_type *uinput_str { + reinterpret_cast(input_str) + }; unsigned int c, c2; const value_type match_one { '?' }; const value_type match_all { '*' }; @@ -2915,28 +2931,31 @@ _like_clause_compare_(const char *pattern, c == match_one) { if (c == match_one && _LikeClauseUtil_::char_read(&uinput_str) == 0) - return (false); + return (like_clause_res::NO_MATCH_INPUT_EXHAUSTED); } if (c == 0) { - return (true); + return (like_clause_res::MATCH); } else if (c == esc_char) { c = _LikeClauseUtil_::char_read(&upattern); if (c == 0) - return (false); + return (like_clause_res::NO_MATCH); } else if (c == match_set) { + like_clause_res rc { like_clause_res::NO_MATCH }; + while (*uinput_str && - _like_clause_compare_( + (rc = _like_clause_compare_impl_( reinterpret_cast(&upattern[-1]), reinterpret_cast(uinput_str), case_insensitive, - esc_char) == 0) { + esc_char)) == like_clause_res::NO_MATCH) { if ((*(uinput_str++)) >= 0xc0) while ((*uinput_str & 0xc0) == 0x80) uinput_str++; } - return (*uinput_str != 0); + return (*uinput_str != 0 + ? rc : like_clause_res::NO_MATCH_INPUT_EXHAUSTED); } while ((c2 = _LikeClauseUtil_::char_read(&uinput_str)) != 0) { if (case_insensitive) { @@ -2952,28 +2971,37 @@ _like_clause_compare_(const char *pattern, c2 = _LikeClauseUtil_::char_read(&uinput_str); } if (c2 == 0) - return (false); - if (_like_clause_compare_( + return (like_clause_res::NO_MATCH_INPUT_EXHAUSTED); + + const like_clause_res rc { + _like_clause_compare_impl_( reinterpret_cast(upattern), reinterpret_cast(uinput_str), case_insensitive, - esc_char)) - return (true); + esc_char) + }; + + // A match, or a failure caused by running out of input. + // In both cases retrying with this '*' consuming more is + // pointless. + // + if (rc != like_clause_res::NO_MATCH) + return (rc); } - return (false); + return (like_clause_res::NO_MATCH_INPUT_EXHAUSTED); } else if (c == match_one && ! prev_escape) { if (_LikeClauseUtil_::char_read(&uinput_str) == 0) - return (false); + return (like_clause_res::NO_MATCH_INPUT_EXHAUSTED); } - else if (c == match_set) { + else if (c == match_set && ! prev_escape) { unsigned int prior_c { 0 }; int seen { 0 }; int invert { 0 }; c = _LikeClauseUtil_::char_read(&uinput_str); if (c == 0) - return (false); + return (like_clause_res::NO_MATCH_INPUT_EXHAUSTED); c2 = _LikeClauseUtil_::char_read(&upattern); if (c2 == '^') { invert = 1; @@ -3002,24 +3030,41 @@ _like_clause_compare_(const char *pattern, c2 = _LikeClauseUtil_::char_read(&upattern); } if (c2 == 0 || (seen ^ invert) == 0) - return (false); + return (like_clause_res::NO_MATCH); } else if (esc_char == c && ! prev_escape) { prev_escape = true; } else { c2 = _LikeClauseUtil_::char_read(&uinput_str); + if (c2 == 0) + return (like_clause_res::NO_MATCH_INPUT_EXHAUSTED); if (case_insensitive) { _LikeClauseUtil_::upper_to_lower(c); _LikeClauseUtil_::upper_to_lower(c2); } if (c != c2) - return (false); + return (like_clause_res::NO_MATCH); prev_escape = false; } } - return (*uinput_str == 0); + return (*uinput_str == 0 + ? like_clause_res::MATCH : like_clause_res::NO_MATCH); +} + +// ---------------------------------------------------------------------------- + +static inline bool +_like_clause_compare_(const char *pattern, + const char *input_str, + bool case_insensitive = false, + unsigned int esc_char = '\\') { + + return (_like_clause_compare_impl_(pattern, + input_str, + case_insensitive, + esc_char) == like_clause_res::MATCH); } // ---------------------------------------------------------------------------- diff --git a/include/DataFrame/Internals/DataFrame_visit.tcc b/include/DataFrame/Internals/DataFrame_visit.tcc index 9c3ca14d..5a287291 100644 --- a/include/DataFrame/Internals/DataFrame_visit.tcc +++ b/include/DataFrame/Internals/DataFrame_visit.tcc @@ -124,11 +124,11 @@ template template V &DataFrame::visit (const char *name, V &visitor, bool in_reverse) { - auto &vec = get_column(name); - const size_type idx_s = indices_.size(); - const size_type min_s = std::min(vec.size(), idx_s); - size_type i = 0; - T nan_val = get_nan(); + auto &vec { get_column(name) }; + const size_type idx_s { indices_.size() }; + const size_type min_s { std::min(vec.size(), idx_s) }; + size_type i { 0 }; + T nan_val { get_nan() }; visitor.pre(); if (! in_reverse) [[likely]] { @@ -138,12 +138,12 @@ V &DataFrame::visit (const char *name, V &visitor, bool in_reverse) { visitor (indices_[i], nan_val); } else { - const size_type diff = idx_s - min_s; - const size_type idx_s_1 = idx_s - 1; + const size_type diff { idx_s - min_s }; + const size_type idx_s_1 { idx_s - 1 }; for (; i < diff; ++i) visitor (indices_[idx_s_1 - i], nan_val); - for (; i < min_s; ++i) + for (; i < idx_s; ++i) visitor (indices_[idx_s_1 - i], vec[idx_s_1 - i]); } visitor.post(); @@ -236,7 +236,7 @@ visit (const char *name1, const char *name2, V &visitor, bool in_reverse) { visitor (indices_[i], ((idx_s_1 - i) < data_s1) ? vec1[idx_s_1 - i] : nan_val1, ((idx_s_1 - i) < data_s2) ? vec2[idx_s_1 - i] : nan_val2); - for (; i < min_s; ++i) [[likely]] + for (; i < idx_s; ++i) [[likely]] visitor (indices_[idx_s_1 - i], vec1[idx_s_1 - i], vec2[idx_s_1 - i]); @@ -353,7 +353,7 @@ visit (const char *name1, ((idx_s_1 - i) < data_s1) ? vec1[idx_s_1 - i] : nan_val1, ((idx_s_1 - i) < data_s2) ? vec2[idx_s_1 - i] : nan_val2, ((idx_s_1 - i) < data_s3) ? vec3[idx_s_1 - i] : nan_val3); - for (; i < min_s; ++i) [[likely]] + for (; i < idx_s; ++i) [[likely]] visitor (indices_[idx_s_1 - i], vec1[idx_s_1 - i], vec2[idx_s_1 - i], @@ -487,7 +487,7 @@ visit (const char *name1, ((idx_s_1 - i) < data_s2) ? vec2[idx_s_1 - i] : nan_val2, ((idx_s_1 - i) < data_s3) ? vec3[idx_s_1 - i] : nan_val3, ((idx_s_1 - i) < data_s4) ? vec4[idx_s_1 - i] : nan_val4); - for (; i < min_s; ++i) [[likely]] + for (; i < idx_s; ++i) [[likely]] visitor (indices_[idx_s_1 - i], vec1[idx_s_1 - i], vec2[idx_s_1 - i], @@ -637,7 +637,7 @@ visit (const char *name1, ((idx_s_1 - i) < data_s3) ? vec3[idx_s_1 - i] : nan_val3, ((idx_s_1 - i) < data_s4) ? vec4[idx_s_1 - i] : nan_val4, ((idx_s_1 - i) < data_s5) ? vec5[idx_s_1 - i] : nan_val5); - for (; i < min_s; ++i) [[likely]] + for (; i < idx_s; ++i) [[likely]] visitor (indices_[idx_s_1 - i], vec1[idx_s_1 - i], vec2[idx_s_1 - i], @@ -741,15 +741,28 @@ visit_async(const char *name1, template template V &DataFrame:: -single_act_visit (const char *name, V &visitor, bool in_reverse) { +single_act_visit(const char *name, V &visitor, bool in_reverse) { - auto &vec = get_column(name); + auto &vec { get_column(name) }; visitor.pre(); if (! in_reverse) [[likely]] visitor (indices_.begin(), indices_.end(), vec.begin(), vec.end()); - else - visitor (indices_.rbegin(), indices_.rend(), vec.rbegin(), vec.rend()); + else { + // In reverse, all ranges must start at the same row. Otherwise, with + // ranges of different lengths, the reversed ranges are misaligned + // relative to each other (i.e. the last index row is paired with the + // last element of a shorter column). So, they all start at the last + // row that is common to all of them. + // + const size_type min_s { + std::min(indices_.size(), vec.size()) + }; + + visitor (indices_.rbegin() + (indices_.size() - min_s), + indices_.rend(), + vec.rbegin() + (vec.size() - min_s), vec.rend()); + } visitor.post(); return (visitor); @@ -810,9 +823,9 @@ single_act_visit (const char *name1, V &visitor, bool in_reverse) { - SpinGuard guard (lock_); - const ColumnVecType &vec1 = get_column(name1, false); - const ColumnVecType &vec2 = get_column(name2, false); + SpinGuard guard { lock_ }; + const ColumnVecType &vec1 { get_column(name1, false) }; + const ColumnVecType &vec2 { get_column(name2, false) }; guard.release(); visitor.pre(); @@ -820,10 +833,15 @@ single_act_visit (const char *name1, visitor (indices_.begin(), indices_.end(), vec1.begin(), vec1.end(), vec2.begin(), vec2.end()); - else - visitor (indices_.rbegin(), indices_.rend(), - vec1.rbegin(), vec1.rend(), - vec2.rbegin(), vec2.rend()); + else { + const size_type min_s = + std::min({ indices_.size(), vec1.size(), vec2.size() }); + + visitor (indices_.rbegin() + (indices_.size() - min_s), + indices_.rend(), + vec1.rbegin() + (vec1.size() - min_s), vec1.rend(), + vec2.rbegin() + (vec2.size() - min_s), vec2.rend()); + } visitor.post(); return (visitor); @@ -899,10 +917,10 @@ single_act_visit (const char *name1, V &visitor, bool in_reverse) { - SpinGuard guard (lock_); - const ColumnVecType &vec1 = get_column(name1, false); - const ColumnVecType &vec2 = get_column(name2, false); - const ColumnVecType &vec3 = get_column(name3, false); + SpinGuard guard { lock_ }; + const ColumnVecType &vec1 { get_column(name1, false) }; + const ColumnVecType &vec2 { get_column(name2, false) }; + const ColumnVecType &vec3 { get_column(name3, false) }; guard.release(); visitor.pre(); @@ -911,11 +929,17 @@ single_act_visit (const char *name1, vec1.begin(), vec1.end(), vec2.begin(), vec2.end(), vec3.begin(), vec3.end()); - else - visitor (indices_.rbegin(), indices_.rend(), - vec1.rbegin(), vec1.rend(), - vec2.rbegin(), vec2.rend(), - vec3.rbegin(), vec3.rend()); + else { + const size_type min_s = + std::min({ indices_.size(), vec1.size(), vec2.size(), + vec3.size() }); + + visitor (indices_.rbegin() + (indices_.size() - min_s), + indices_.rend(), + vec1.rbegin() + (vec1.size() - min_s), vec1.rend(), + vec2.rbegin() + (vec2.size() - min_s), vec2.rend(), + vec3.rbegin() + (vec3.size() - min_s), vec3.rend()); + } visitor.post(); return (visitor); @@ -1016,12 +1040,18 @@ single_act_visit (const char *name1, vec2.begin(), vec2.end(), vec3.begin(), vec3.end(), vec4.begin(), vec4.end()); - else - visitor (indices_.rbegin(), indices_.rend(), - vec1.rbegin(), vec1.rend(), - vec2.rbegin(), vec2.rend(), - vec3.rbegin(), vec3.rend(), - vec4.rbegin(), vec4.rend()); + else { + const size_type min_s = + std::min({ indices_.size(), vec1.size(), vec2.size(), + vec3.size(), vec4.size() }); + + visitor (indices_.rbegin() + (indices_.size() - min_s), + indices_.rend(), + vec1.rbegin() + (vec1.size() - min_s), vec1.rend(), + vec2.rbegin() + (vec2.size() - min_s), vec2.rend(), + vec3.rbegin() + (vec3.size() - min_s), vec3.rend(), + vec4.rbegin() + (vec4.size() - min_s), vec4.rend()); + } visitor.post(); return (visitor); @@ -1133,13 +1163,19 @@ single_act_visit (const char *name1, vec3.begin(), vec3.end(), vec4.begin(), vec4.end(), vec5.begin(), vec5.end()); - else - visitor (indices_.rbegin(), indices_.rend(), - vec1.rbegin(), vec1.rend(), - vec2.rbegin(), vec2.rend(), - vec3.rbegin(), vec3.rend(), - vec4.rbegin(), vec4.rend(), - vec5.rbegin(), vec5.rend()); + else { + const size_type min_s = + std::min({ indices_.size(), vec1.size(), vec2.size(), + vec3.size(), vec4.size(), vec5.size() }); + + visitor (indices_.rbegin() + (indices_.size() - min_s), + indices_.rend(), + vec1.rbegin() + (vec1.size() - min_s), vec1.rend(), + vec2.rbegin() + (vec2.size() - min_s), vec2.rend(), + vec3.rbegin() + (vec3.size() - min_s), vec3.rend(), + vec4.rbegin() + (vec4.size() - min_s), vec4.rend(), + vec5.rbegin() + (vec5.size() - min_s), vec5.rend()); + } visitor.post(); return (visitor); diff --git a/include/DataFrame/Utils/Matrix.tcc b/include/DataFrame/Utils/Matrix.tcc index 2c6a2f97..32a1f289 100644 --- a/include/DataFrame/Utils/Matrix.tcc +++ b/include/DataFrame/Utils/Matrix.tcc @@ -2492,8 +2492,7 @@ ldlt(std::vector &D, MA &L) const { sum += l_tmp(k, j) * l_tmp(k, j) * d_tmp[j]; d_tmp[k] = at(k, k) - sum; - if (std::abs(d_tmp[k]) < std::numeric_limits::epsilon()) - [[unlikely]] + if (std::abs(d_tmp[k]) < std::numeric_limits::epsilon()) [[unlikely]] throw NotFeasible("Matrix::ldlt(): Matrix is singular"); // Compute L(i,k) for i = k+1 .. rows-1 @@ -3661,8 +3660,8 @@ Matrix Matrix:: get_random(size_type rows, size_type cols, T low, T high, unsigned int seed) requires std::floating_point { - Matrix result; - RandGenParams rp; + Matrix result; + RandGenParams rp; rp.min_value = low; rp.max_value = high; diff --git a/include/DataFrame/Utils/Threads/ThreadPool.tcc b/include/DataFrame/Utils/Threads/ThreadPool.tcc index 72c704f4..9deea1a2 100644 --- a/include/DataFrame/Utils/Threads/ThreadPool.tcc +++ b/include/DataFrame/Utils/Threads/ThreadPool.tcc @@ -334,7 +334,7 @@ ThreadPool::parallel_sort(const I begin, const I end, P compare) { if (begin >= end) return; - const size_type data_size = std::distance(begin, end); + const size_type data_size { size_type(std::distance(begin, end)) }; if (data_size <= TH) { std::sort(begin, end, compare); @@ -343,32 +343,51 @@ ThreadPool::parallel_sort(const I begin, const I end, P compare) { // Pivot selection (median‑of‑three) // - auto mid { begin + (data_size / 2) }; - auto pivot_it { _median_of_three_(begin, mid, end - 1, compare) }; - const auto pivot { *pivot_it }; + auto mid { begin + (data_size / 2) }; + auto pivot_it { _median_of_three_(begin, mid, end - 1, compare) }; + + // The pivot must be a value copy, not a reference into the range. With + // proxy iterators (e.g. std::ranges::views::zip), *pivot_it is a tuple of + // references that would keep aliasing the slot the pivot is swapped out + // of, so the pivot would change under the partition and the result would + // be mis-sorted. + // + const std::iter_value_t pivot { *pivot_it }; std::iter_swap(pivot_it, end - 1); // Move pivot to end ‑ 1 - auto cut { + // Three-way partition: + // [begin, lt) < pivot, [lt, eq) == pivot, [eq, end - 1) > pivot. + // Elements equal to the pivot are never recursed into. With a two-way + // partition, data with many duplicate keys shrinks by one element per + // level: recursion depth O(n), quadratic time, and stack overflow. + // + const auto lt { std::ranges::partition(begin, end - 1, [&pivot, &compare](const auto &x) -> bool { return (compare(x, pivot)); - }) + }).begin() + }; + const auto eq { + std::ranges::partition(lt, end - 1, + [&pivot, &compare](const auto &x) -> bool { + return (! compare(pivot, x)); + }).begin() }; - std::iter_swap(cut.begin(), end - 1); // Restore pivot + std::iter_swap(eq, end - 1); // Restore pivot to the end of equal block auto lf { dispatch(false, &ThreadPool::parallel_sort, this, begin, - cut.begin(), + lt, compare) }; auto rf { dispatch(false, &ThreadPool::parallel_sort, this, - cut.begin() + 1, + eq + 1, end, compare) }; diff --git a/include/DataFrame/Vectors/HeteroConstPtrView.h b/include/DataFrame/Vectors/HeteroConstPtrView.h index c65df293..8eba0d59 100644 --- a/include/DataFrame/Vectors/HeteroConstPtrView.h +++ b/include/DataFrame/Vectors/HeteroConstPtrView.h @@ -71,12 +71,12 @@ struct HeteroConstPtrView { template HeteroConstPtrView(VectorConstPtrView &&vec); HeteroConstPtrView(const HeteroConstPtrView &that); - HeteroConstPtrView(HeteroConstPtrView &&that); + HeteroConstPtrView(HeteroConstPtrView &&that) noexcept; ~HeteroConstPtrView() { clear(); } HeteroConstPtrView &operator= (const HeteroConstPtrView &rhs); - HeteroConstPtrView &operator= (HeteroConstPtrView &&rhs); + HeteroConstPtrView &operator= (HeteroConstPtrView &&rhs) noexcept; template [[nodiscard]] VectorConstPtrView &get_vector(); diff --git a/include/DataFrame/Vectors/HeteroConstPtrView.tcc b/include/DataFrame/Vectors/HeteroConstPtrView.tcc index e69d2c81..7b1526d4 100644 --- a/include/DataFrame/Vectors/HeteroConstPtrView.tcc +++ b/include/DataFrame/Vectors/HeteroConstPtrView.tcc @@ -276,9 +276,8 @@ HeteroConstPtrView::HeteroConstPtrView (const HeteroConstPtrView &that) { *this = that; } template -HeteroConstPtrView::HeteroConstPtrView (HeteroConstPtrView &&that) { - *this = that; -} +HeteroConstPtrView:: +HeteroConstPtrView (HeteroConstPtrView &&that) noexcept { *this = that; } // ---------------------------------------------------------------------------- @@ -302,7 +301,7 @@ operator= (const HeteroConstPtrView &rhs) { template HeteroConstPtrView &HeteroConstPtrView:: -operator= (HeteroConstPtrView &&rhs) { +operator= (HeteroConstPtrView &&rhs) noexcept { if (&rhs != this) [[likely]] { clear(); diff --git a/include/DataFrame/Vectors/HeteroConstView.h b/include/DataFrame/Vectors/HeteroConstView.h index 1848d1dc..7183cb7b 100644 --- a/include/DataFrame/Vectors/HeteroConstView.h +++ b/include/DataFrame/Vectors/HeteroConstView.h @@ -67,12 +67,12 @@ struct HeteroConstView { void set_empty_vec(); HeteroConstView(const HeteroConstView &that); - HeteroConstView(HeteroConstView &&that); + HeteroConstView(HeteroConstView &&that) noexcept; ~HeteroConstView() { clear(); } HeteroConstView &operator= (const HeteroConstView &rhs); - HeteroConstView &operator= (HeteroConstView &&rhs); + HeteroConstView &operator= (HeteroConstView &&rhs) noexcept; template [[nodiscard]] VectorConstView &get_vector(); diff --git a/include/DataFrame/Vectors/HeteroConstView.tcc b/include/DataFrame/Vectors/HeteroConstView.tcc index 204e58a7..869dfa73 100644 --- a/include/DataFrame/Vectors/HeteroConstView.tcc +++ b/include/DataFrame/Vectors/HeteroConstView.tcc @@ -250,7 +250,8 @@ template HeteroConstView:: HeteroConstView(const HeteroConstView &that) { *this = that; } template -HeteroConstView::HeteroConstView(HeteroConstView &&that) { *this = that; } +HeteroConstView:: +HeteroConstView(HeteroConstView &&that) noexcept { *this = that; } // ---------------------------------------------------------------------------- @@ -273,7 +274,8 @@ operator= (const HeteroConstView &rhs) { // ---------------------------------------------------------------------------- template -HeteroConstView &HeteroConstView::operator= (HeteroConstView &&rhs) { +HeteroConstView &HeteroConstView:: +operator= (HeteroConstView &&rhs) noexcept { if (&rhs != this) [[likely]] { clear(); diff --git a/include/DataFrame/Vectors/HeteroPtrView.h b/include/DataFrame/Vectors/HeteroPtrView.h index 688fecea..681f92d1 100644 --- a/include/DataFrame/Vectors/HeteroPtrView.h +++ b/include/DataFrame/Vectors/HeteroPtrView.h @@ -71,12 +71,12 @@ struct HeteroPtrView { template HeteroPtrView(VectorPtrView &&vec); HeteroPtrView(const HeteroPtrView &that); - HeteroPtrView(HeteroPtrView &&that); + HeteroPtrView(HeteroPtrView &&that) noexcept; ~HeteroPtrView() { clear(); } HeteroPtrView &operator= (const HeteroPtrView &rhs); - HeteroPtrView &operator= (HeteroPtrView &&rhs); + HeteroPtrView &operator= (HeteroPtrView &&rhs) noexcept; template [[nodiscard]] VectorPtrView &get_vector(); diff --git a/include/DataFrame/Vectors/HeteroPtrView.tcc b/include/DataFrame/Vectors/HeteroPtrView.tcc index daa5a0f8..f218e846 100644 --- a/include/DataFrame/Vectors/HeteroPtrView.tcc +++ b/include/DataFrame/Vectors/HeteroPtrView.tcc @@ -356,7 +356,8 @@ HeteroPtrView::HeteroPtrView() = default; template HeteroPtrView::HeteroPtrView (const HeteroPtrView &that) { *this = that; } template -HeteroPtrView::HeteroPtrView (HeteroPtrView &&that) { *this = that; } +HeteroPtrView:: +HeteroPtrView (HeteroPtrView &&that) noexcept { *this = that; } // ---------------------------------------------------------------------------- @@ -378,7 +379,7 @@ HeteroPtrView &HeteroPtrView::operator= (const HeteroPtrView &rhs) { // ---------------------------------------------------------------------------- template -HeteroPtrView &HeteroPtrView::operator= (HeteroPtrView &&rhs) { +HeteroPtrView &HeteroPtrView::operator= (HeteroPtrView &&rhs) noexcept { if (&rhs != this) [[likely]] { clear(); diff --git a/include/DataFrame/Vectors/HeteroVector.h b/include/DataFrame/Vectors/HeteroVector.h index b1ef8a03..0ffae54d 100644 --- a/include/DataFrame/Vectors/HeteroVector.h +++ b/include/DataFrame/Vectors/HeteroVector.h @@ -59,12 +59,12 @@ struct HeteroVector { HeteroVector(); HeteroVector(const HeteroVector &that); - HeteroVector(HeteroVector &&that); + HeteroVector(HeteroVector &&that) noexcept; ~HeteroVector() { clear(); } HeteroVector &operator= (const HeteroVector &rhs); - HeteroVector &operator= (HeteroVector &&rhs); + HeteroVector &operator= (HeteroVector &&rhs) noexcept; template [[nodiscard]] std::vector::type> & diff --git a/include/DataFrame/Vectors/HeteroVector.tcc b/include/DataFrame/Vectors/HeteroVector.tcc index a312f6fd..f14540e6 100644 --- a/include/DataFrame/Vectors/HeteroVector.tcc +++ b/include/DataFrame/Vectors/HeteroVector.tcc @@ -383,7 +383,7 @@ HeteroVector::HeteroVector () { template HeteroVector:: HeteroVector (const HeteroVector &that) { *this = that; } template HeteroVector:: -HeteroVector (HeteroVector &&that) { *this = std::move(that); } +HeteroVector (HeteroVector &&that) noexcept { *this = std::move(that); } // ---------------------------------------------------------------------------- @@ -406,7 +406,7 @@ HeteroVector &HeteroVector::operator= (const HeteroVector &rhs) { // ---------------------------------------------------------------------------- template -HeteroVector &HeteroVector::operator= (HeteroVector &&rhs) { +HeteroVector &HeteroVector::operator= (HeteroVector &&rhs) noexcept { if (&rhs != this) [[likely]] { clear(); diff --git a/include/DataFrame/Vectors/HeteroView.h b/include/DataFrame/Vectors/HeteroView.h index 6f3b1469..3967f182 100644 --- a/include/DataFrame/Vectors/HeteroView.h +++ b/include/DataFrame/Vectors/HeteroView.h @@ -67,12 +67,12 @@ struct HeteroView { void set_empty_vec(); HeteroView(const HeteroView &that); - HeteroView(HeteroView &&that); + HeteroView(HeteroView &&that) noexcept; ~HeteroView() { clear(); } HeteroView &operator= (const HeteroView &rhs); - HeteroView &operator= (HeteroView &&rhs); + HeteroView &operator= (HeteroView &&rhs) noexcept; template [[nodiscard]] VectorView &get_vector(); diff --git a/include/DataFrame/Vectors/HeteroView.tcc b/include/DataFrame/Vectors/HeteroView.tcc index 39740d63..0a41f966 100644 --- a/include/DataFrame/Vectors/HeteroView.tcc +++ b/include/DataFrame/Vectors/HeteroView.tcc @@ -359,7 +359,7 @@ HeteroView::HeteroView() = default; template HeteroView::HeteroView (const HeteroView &that) { *this = that; } template -HeteroView::HeteroView (HeteroView &&that) { *this = that; } +HeteroView::HeteroView (HeteroView &&that) noexcept { *this = that; } // ---------------------------------------------------------------------------- @@ -381,7 +381,7 @@ HeteroView &HeteroView::operator= (const HeteroView &rhs) { // ---------------------------------------------------------------------------- template -HeteroView &HeteroView::operator= (HeteroView &&rhs) { +HeteroView &HeteroView::operator= (HeteroView &&rhs) noexcept { if (&rhs != this) [[likely]] { clear(); diff --git a/include/DataFrame/Vectors/VectorPtrView.h b/include/DataFrame/Vectors/VectorPtrView.h index 22771f24..2cb9e007 100644 --- a/include/DataFrame/Vectors/VectorPtrView.h +++ b/include/DataFrame/Vectors/VectorPtrView.h @@ -117,7 +117,7 @@ class VectorPtrView { inline VectorPtrView () = default; inline VectorPtrView (const VectorPtrView &) = default; - inline VectorPtrView (VectorPtrView &&) = default; + inline VectorPtrView (VectorPtrView &&) noexcept = default; inline VectorPtrView(const std::vector &rhs) { *this = rhs; } inline VectorPtrView(std::vector &rhs) { *this = rhs; } template @@ -150,7 +150,7 @@ class VectorPtrView { } inline VectorPtrView &operator = (const VectorPtrView &) = default; - inline VectorPtrView &operator = (VectorPtrView &&) = default; + inline VectorPtrView &operator = (VectorPtrView &&) noexcept = default; VectorPtrView &operator = (const std::vector &rhs) = delete; inline VectorPtrView &operator = (std::vector &rhs) { @@ -635,7 +635,7 @@ class VectorConstPtrView { inline VectorConstPtrView () = default; inline VectorConstPtrView (const VectorConstPtrView &) = default; - inline VectorConstPtrView (VectorConstPtrView &&) = default; + inline VectorConstPtrView (VectorConstPtrView &&) noexcept = default; inline VectorConstPtrView(const std::vector &rhs) { *this = rhs; } inline VectorConstPtrView(std::vector &rhs) { *this = rhs; } template @@ -669,7 +669,8 @@ class VectorConstPtrView { inline VectorConstPtrView & operator = (const VectorConstPtrView &) = default; - inline VectorConstPtrView &operator = (VectorConstPtrView &&) = default; + inline VectorConstPtrView & + operator = (VectorConstPtrView &&) noexcept = default; inline VectorConstPtrView &operator = (const std::vector &rhs) { VectorConstPtrView tmp_vec; diff --git a/include/DataFrame/Vectors/VectorView.h b/include/DataFrame/Vectors/VectorView.h index 20848b8b..2ac972bc 100644 --- a/include/DataFrame/Vectors/VectorView.h +++ b/include/DataFrame/Vectors/VectorView.h @@ -67,9 +67,9 @@ class VectorView { VectorView() = default; VectorView(const VectorView &) = default; - VectorView(VectorView &&) = default; + VectorView(VectorView &&) noexcept = default; VectorView &operator= (const VectorView &) = default; - VectorView &operator= (VectorView &&) = default; + VectorView &operator= (VectorView &&) noexcept = default; ~VectorView() = default; inline VectorView (value_type *bp, value_type *ep) noexcept @@ -503,13 +503,13 @@ class VectorConstView { VectorConstView() = default; VectorConstView(const VectorConstView &) = default; - VectorConstView(VectorConstView &&) = default; + VectorConstView(VectorConstView &&) noexcept = default; VectorConstView &operator= (const VectorConstView &) = default; - VectorConstView &operator= (VectorConstView &&) = default; + VectorConstView &operator= (VectorConstView &&) noexcept = default; ~VectorConstView() = default; inline - VectorConstView (const value_type *bp, const value_type *ep) noexcept + VectorConstView(const value_type *bp, const value_type *ep) noexcept : begin_ptr_(bp), end_ptr_(ep) { } // The purpose of this method is for the user be able to conform to STL diff --git a/test/dataframe_tester_3.cc b/test/dataframe_tester_3.cc index 7585fbf9..91f0d731 100644 --- a/test/dataframe_tester_3.cc +++ b/test/dataframe_tester_3.cc @@ -227,6 +227,35 @@ static void test_concat_view() { assert(const_result3.get_column("dbl_col_2")[0] == 100.0); assert(const_result3.get_column("dbl_col_2")[5] == 105.0); assert(const_result3.get_column("dbl_col_2")[10] == 112.0); + + // The two data frames have different lengths, and the rhs is the longer + // one. All of the rhs rows must show up in the view. (This used to be + // truncated to the length of the lhs column.) + // + { + MyDataFrame short_df; + MyDataFrame long_df; + + short_df.load_index(MyDataFrame::gen_sequence_index(0, 5, 1)); + long_df.load_index(MyDataFrame::gen_sequence_index(1000, 1008, 1)); + short_df.load_column("dbl_col", { 0.0, 1.0, 2.0, 3.0, 4.0 }); + long_df.load_column("dbl_col", + { 1000.0, 1001.0, 1002.0, 1003.0, + 1004.0, 1005.0, 1006.0, 1007.0 }); + + auto result4 { + short_df.concat_view + (long_df, concat_policy::all_columns) + }; + const auto &col { result4.get_column("dbl_col") }; + + assert(result4.get_index().size() == 13); + assert(col.size() == 13); + assert(col[0] == 0.0); + assert(col[4] == 4.0); + assert(col[5] == 1000.0); + assert(col[12] == 1007.0); + } } // ---------------------------------------------------------------------------- diff --git a/test/dataframe_tester_4.cc b/test/dataframe_tester_4.cc index 9aca6ac3..9557b588 100644 --- a/test/dataframe_tester_4.cc +++ b/test/dataframe_tester_4.cc @@ -965,6 +965,38 @@ static void test_difference() { assert(! diff_df.has_column("self_dbl_col_2")); assert(! diff_df.has_column("other_dbl_col_2")); assert(! diff_df.has_column("dbl_col_2")); + + // Columns of different lengths, with the last difference in another column + // further down. The tail rows must stay aligned with their rows. + // + { + MyDataFrame lhs; + MyDataFrame rhs; + + lhs.load_index(MyDataFrame::gen_sequence_index(0, 6, 1)); + rhs.load_index(MyDataFrame::gen_sequence_index(0, 6, 1)); + lhs.load_column("x", { 1, 2, 3, 4, 5, 6 }); + rhs.load_column("x", { 1, 9, 3, 4 }, + nan_policy::dont_pad_with_nans); + lhs.load_column("y", { 10, 11, 12, 13, 14, 15 }); + rhs.load_column("y", { 10, 11, 12, 13, 99, 15 }); + + const auto res = lhs.difference(rhs); + + assert(res.get_index().size() == 6); + + const auto &self_x = res.get_column("self_x"); + + assert(self_x.size() == 6); + assert(std::isnan(self_x[0])); + assert(self_x[1] == 2.0); + assert(std::isnan(self_x[2])); + assert(std::isnan(self_x[3])); + assert(self_x[4] == 5.0); + assert(self_x[5] == 6.0); + assert(res.get_column("self_y")[4] == 14.0); + assert(res.get_column("other_y")[4] == 99.0); + } } // ---------------------------------------------------------------------------- @@ -4234,13 +4266,11 @@ static void test_remove_data_by_fft() { assert((ibm.get_column("IBM_Open").size() == 5031)); assert((ibm_view.get_column("IBM_Open").size() == 5031)); - ibm.remove_data_by_fft("IBM_Close", 1000, 250, - normalization_type::z_score); - assert((ibm.get_column("IBM_Open").size() == (5031 - 3))); + ibm.remove_data_by_fft("IBM_Close", 1000, 80); + assert((ibm.get_column("IBM_Open").size() == (5031 - 9))); - ibm_view.remove_data_by_fft - ("IBM_Close", 1000, 250, normalization_type::z_score); - assert((ibm_view.get_column("IBM_Open").size() == (5031 - 3))); + ibm_view.remove_data_by_fft("IBM_Close", 1000, 80); + assert((ibm_view.get_column("IBM_Open").size() == (5031 - 9))); } // ---------------------------------------------------------------------------- @@ -4757,16 +4787,16 @@ static void test_detect_and_change() { { "IBM_Close", "IBM_Open" }, detect_method::fft, fill_policy::mid_point, - { .threshold = 250.0, - .norm_type = normalization_type::z_score, + { .threshold = 80.0, + .norm_type = normalization_type::none, .freq_num = 1000 }); - assert((std::fabs(close_col[502] - 82.02) < 0.01)); - assert((std::fabs(close_col[1001] - 89.805) < 0.01)); - assert((std::fabs(close_col[2002] - 88.055) < 0.01)); - assert((std::fabs(open_col[2] - 1.0) < 0.01)); // It didn't catch it - assert((std::fabs(open_col[3000] - 2.5) < 0.01)); // It didn't catch it - assert((std::fabs(open_col[5029] - 108.28) < 0.01)); + assert((std::fabs(close_col[502] - 260.87) < 0.001)); + assert((std::fabs(close_col[1001] - 292.555) < 0.001)); + assert((std::fabs(close_col[2002] - 276.995) < 0.001)); + assert((std::fabs(open_col[2] - 1.0) < 0.01)); // Didn't catch this + assert((std::fabs(open_col[3000] - 210.15) < 0.01)); + assert((std::fabs(open_col[5029] - 294.488) < 0.01)); } // Now we need a DataFrame with a numeric index to be able to use diff --git a/test/dataframe_tester_5.cc b/test/dataframe_tester_5.cc index cebd62d2..33867616 100644 --- a/test/dataframe_tester_5.cc +++ b/test/dataframe_tester_5.cc @@ -7078,6 +7078,33 @@ static void test_StreamAppender() { // ----------------------------------------------------------------------------- +static void test_parallel_sort() { + + std::cout << "\nTesting parallel_sort( ) ..." << std::endl; + + constexpr std::size_t N { 500'009 }; + std::vector data(N); + + std::srand(123); + for (int &item : data) item = std::rand(); + + // Let's make it so it has a lot of duplicates + // + for (std::size_t i { 0 }; i < 1000; ++i) { + data[std::rand() % N] = 1000; + } + + ThreadPool tp { 6 }; // 6 threads + + tp.parallel_sort(data.begin(), data.end()); + // std::sort(data.begin(), data.end()); + + for (std::size_t i { 1 }; i < N; ++i) + assert(data[i - 1] <= data[i]); +} + +// ----------------------------------------------------------------------------- + int main(int, char *[]) { ULDataFrame::set_optimum_thread_level(); @@ -7127,6 +7154,7 @@ int main(int, char *[]) { test_read_chunked_data(); test_streamed_write(); test_StreamAppender(); + test_parallel_sort(); return (0); } diff --git a/test/dataframe_tester_output.txt b/test/dataframe_tester_output.txt index 3ed6c937..fce2d504 100644 --- a/test/dataframe_tester_output.txt +++ b/test/dataframe_tester_output.txt @@ -720,12 +720,12 @@ col_2:1::9, col_3:1::16, col_str:1::22, col_4:1::23, -INDEX:1::123452, -col_1:1::3, -col_2:1::10, -col_3:1::17, -col_str:1::33, -col_4:1::24, +INDEX:1::123450, +col_1:1::1, +col_2:1::8, +col_3:1::15, +col_str:1::11, +col_4:1::22, Testing write(json) ... Writing in JSON: @@ -768,13 +768,13 @@ Means of clusters are: 19.9820684374, 3.85685151881, 0.606906002983, 1.840001468 4.39684180804 | 1.12423086285, 5.32645061833 | 0.258879189718, 6.56770612254 | 1.88980716816, 4.97200415894 | 0.407118210797, 6.03564845001 | 0.751012545119, 2.94035843782 | 0.314775456319, 3.60858208918 | 2.8089375563, 2.85547872035 | 2.51220235177, 3.89964990999 | 0.856374176369, 3.02735723184 | 0.226780241358, 3.11375978646 | 1.37885008492, 3.23563684881 | 1.32453935853, 4.74247908133 | 2.89363607052, 3.96514946019 | 0.387975516108, 4.4204216991 | 0.657322986031, 5.4094749177 | 0.376668387939, 3.21957447237 | 2.63364446483, 3.3948416627 | 0.163593376023, 3.84539096788 | 0.297100044069, 5.92619382132 | 1.84933460179, 4.590537026 | 2.53226329642, 3.06375316017 | 1.05043548764, 5.07407491355 | 2.45226502214, 6.28652144612 | 0.975087587592, 6.58207129189 | 0.355147337604, 3.72469838853 | 1.61156994808, 6.52684709038 | 1.96726515223, 2.9631288797 | 0.857151949309, 2.97138456376 | 0.629994862143, 5.90181207884 | 1.652460374, 3.05207906639 | 0.495742043608, 2.89162903925 | 1.03099415637, 2.75375629284 | 1.33757933345, 4.2123939732 | 2.11995054879, 4.7521600173 | 1.26866382559, 9.20566066884 | 0.635108361464, 3.35937247059 | 0.237096980877, 6.7470372648 | 2.90489612946, 5.56776407333 | 1.45780412902, 2.97283115952 | 0.768662732119, 5.29033890732 | 1.46684775613, 3.33442909318 | 1.41208702577, 3.3456478028 | 3.15791207885, 5.34911249975 | 0.310107651423, 6.26122042314 | 0.228863156957, 5.58341660812 | 0.652454865698, 4.0839718193 | 0.433244844548, 3.4840279873 | 0.588712471498, 5.48604913149 | 1.29273094631, 2.69170609551 | 0.557907104774, 4.07594518419 | 0.0794660417283, 4.22286388466 | 2.27121671294, 2.78656421487 | 1.09789223697, 3.68130821292 | 0.42822664425, 5.99835163287 | 1.64461791151, 3.77323745304 | 2.3901193323, 4.28719972597 | 0.741702952331, 3.48535661343 | 0.547992783008, 2.91570692268 | 0.297979629089, 4.22482418509 | 1.60766663614, 2.87894041417 | 0.210360178912, 3.81005611699 | 2.09964844162, 5.79399808209 | 0.665058152339, 4.83290584104 | 0.917001805312, 4.54824347301 | 1.28630290164, 2.7904401791 | 0.303055664379, 4.08422388761 | 1.52920833284, 4.597008972 | 0.552201184549, 3.44381095773 | 0.166042909241, 3.50438824207 | 1.92655877373, 4.00278453337 | 0.724330335054, 4.13312479796 | 3.83162584598, 2.98368298402 | 0.248108338937, 2.99413350468 | 0.106464616772, 4.57993344663 | 2.78540985725, 7.41604024389 | 0.452783882865, 5.34126437112 | 0.222824052156, 3.28587796229 | 0.595307265723, 3.12354995598 | 0.167846498981, 3.3611648224 | 0.838456623147, 3.42429119576 | 0.789557072651, 5.8512550284 | 1.99281532804, 3.91105879269 | 0.452095791245, 3.69740704803 | 0.820050676208, 3.1442635652 | 2.21862308472, 2.92284337303 | 0.620210327906, 3.36458898403 | 0.714739708796, 3.19489739931 | 0.350048969031, 7.88833318096 | 0.365468571658, 5.00308578152 | 0.223766784791, 3.22840497378 | 0.539154032626, 4.32414826891 | 0.342917766201, 4.2372602031 | 2.62312171147, 4.03489667364 | 1.19724040465, 2.98468107894 | 1.70816624101, 3.70061721998 | 0.144003322043, 3.32759851079 | 3.22759915673, 3.5348883024 | 2.26666238419, 3.61986027727 | 1.42234783125, 3.13105489274 | 0.42684491591, 4.13584607426 | 0.38240113167, 4.35065180729 | 0.450662984793, 5.29386720306 | 1.13522472017, 4.92212961708 | 0.504918122593, 4.19696666531 | 0.542106242523, 4.70822553306 | 0.431753357932, 4.70496853934 | 0.585882562077, 2.82163939708 | 0.354211377662, 2.7749285673 | 0.773819372569, 4.20106078629 | 3.5468186015, 3.25806230422 | 1.16446490776, 2.97824153046 | 0.634719609118, 6.24049625894 | 0.604503799466, 7.18692448088 | 0.702479476078, 3.9767986394 | 2.46691462078, 4.90972585114 | 2.12411456206, 3.25824555983 | 1.42599273813, 3.1842857736 | 0.780409710085, 7.33108959719 | 0.948593648814, 6.88914480747 | 1.18840666953, 3.93008795772 | 1.03634566272, 4.86100091822 | 0.460655730352, 5.64255515254 | 0.727288715906, 4.05696325215 | 0.596443341098, 4.12078782301 | 0.766925538593, 6.46611201826 | 1.85524070679, 4.94726132676 | 1.97043281815, 6.83805063598 | 0.444160900675, 4.86890068342 | 0.625975194945, 6.76331265223 | 0.224926532447, 8.96799021152 | 1.90689894212, 3.16719895489 | 0.646323589286, 4.93474307859 | 1.07218770687, 4.37542292896 | 0.884930238015, 5.81515843154 | 5.06933221904, -3[-8.73774733197|-1.93743664834|11.0190319172] +3[-10.5353759989|-3.05940794706|-10.1850397707] -3[5.71303266874|-11.1714075284|-5.73628993344] +3[1.87040726365|-11.083196429|7.75581180591] -3[-7.65801427656|6.70019604884|-10.3360783645] +3[-3.48507122354|10.1283967235|10.1481327499] -3[11.212359209|9.74359344371|4.1733220999] +3[10.7082824469|5.32048723657|-8.35012040892] Testing affinity propagation visitor ... @@ -1706,7 +1706,7 @@ INDEX:9:,IBM_Open:9:,IBM_High:9:,IBM_Low:9:,IBM_ COUNT,1721,1721,1721,1721,1721,1721 MISSING,0,0,0,0,0,0 MEAN,150.847588639,151.976879643,149.709529212,150.839442296,129.527833117,4560209.58745 -STD,19.9912242114,19.91396027,20.1165782354,20.0127428788,11.1522066877,5068928 +STD,19.9912242114,19.91396027,20.1165782354,20.0127428788,11.1522066877,2645518.25477 MIN,94.599998,97.739998,90.559998,94.769997,92.30722,1193000 MAX,198.050003,199.210007,195.880005,197.770004,155.360657,30490200 25%,137.6949995,138.720001,136.794998,137.794998,123.625126,3097700 @@ -3080,3 +3080,5 @@ Testing read_chunked_data( ) ... Testing streamed_write( ) ... Testing StreamAppender{ } ... + +Testing parallel_sort( ) ...