diff --git a/crates/fidc-core/src/data.rs b/crates/fidc-core/src/data.rs index 8f6f838..0081a18 100644 --- a/crates/fidc-core/src/data.rs +++ b/crates/fidc-core/src/data.rs @@ -4644,25 +4644,23 @@ fn build_factor_market_cap_order( symbol_ids.len(), "factor rows and symbol ids diverged for {date}" ); - let mut row_indices = rows + let mut keys = rows .iter() - .enumerate() - .filter_map(|(index, row)| { + .zip(symbol_ids) + .filter_map(|(row, symbol_id)| { let market_cap_bn = decision_market_cap_bn(row); - (market_cap_bn.is_finite() && market_cap_bn > 0.0).then_some(index) + (market_cap_bn.is_finite() && market_cap_bn > 0.0) + .then_some((market_cap_bn, *symbol_id)) }) .collect::>(); - row_indices.sort_by(|left, right| { - let left = &rows[*left]; - let right = &rows[*right]; - decision_market_cap_bn(left) - .partial_cmp(&decision_market_cap_bn(right)) - .unwrap_or(std::cmp::Ordering::Equal) - .then_with(|| left.symbol.cmp(&right.symbol)) + // Symbol IDs follow the complete lexical symbol order. Equal keys + // emit identical IDs, so stability does not affect the output. + keys.sort_unstable_by(|left, right| { + left.0.total_cmp(&right.0).then_with(|| left.1.cmp(&right.1)) }); - let ordered = row_indices + let ordered = keys .into_iter() - .map(|index| symbol_ids[index]) + .map(|(_, symbol_id)| symbol_id) .collect::>(); (*date, ordered) }) @@ -5099,6 +5097,79 @@ fn instrument_passes_baseline_selection(instrument: Option<&Instrument>, date: N mod tests { use super::*; + fn reference_market_cap_order( + rows: &[DailyFactorSnapshot], + symbol_ids: &[u32], + ) -> Vec { + let mut indices = (0..rows.len()) + .filter(|index| { + let cap = decision_market_cap_bn(&rows[*index]); + cap.is_finite() && cap > 0.0 + }) + .collect::>(); + indices.sort_by(|left, right| { + decision_market_cap_bn(&rows[*left]) + .partial_cmp(&decision_market_cap_bn(&rows[*right])) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| rows[*left].symbol.cmp(&rows[*right].symbol)) + }); + indices.into_iter().map(|index| symbol_ids[index]).collect() + } + + #[test] + fn compact_market_cap_keys_preserve_reference_order_and_source_values() { + let start = NaiveDate::from_ymd_opt(2024, 1, 2).unwrap(); + let caps = [ + f64::NAN, -0.0, 0.0, f64::NEG_INFINITY, f64::INFINITY, -1.0, + f64::from_bits(1), f64::MIN_POSITIVE, 1.0, 1.0000000000000002, + 1.0, f64::MAX, + ]; + let groups = (0..5).map(|day| { + let date = start + chrono::Duration::days(day); + let mut rows = (0..4096).map(|index| DailyFactorSnapshot { + date, + symbol: format!("instrument_{index:05}_long_non_equity_identity").into(), + market_cap_bn: caps[(index * 17 + day as usize) % caps.len()], + free_float_cap_bn: 1.0, pe_ttm: 1.0, + turnover_ratio: Some(-0.0), effective_turnover_ratio: None, + adjustment_factor_backward1: Some(1.0), + extra_factors: NumericFactorMap::new(), + }).collect::>(); + rows.reverse(); + rows.push(rows[19].clone()); + (date, rows) + }).chain([(start + chrono::Duration::days(5), Vec::new())]) + .collect::>(); + let symbol_index = build_symbol_id_index( + &HashMap::new(), &BTreeMap::new(), &groups, &BTreeMap::new(), + ); + let ids = groups.iter().map(|(date, rows)| (*date, rows.iter() + .map(|row| symbol_index[row.symbol.as_str()]).collect::>())) + .collect::>(); + let expected = groups.iter().map(|(date, rows)| + (*date, reference_market_cap_order(rows, &ids[date]))) + .collect::>(); + let bits = || groups.values().flatten().map(|row| + (row.market_cap_bn.to_bits(), row.turnover_ratio.unwrap().to_bits())) + .collect::>(); + let before = bits(); + for threads in [1, 2, 8] { + let pool = rayon::ThreadPoolBuilder::new().num_threads(threads).build().unwrap(); + assert_eq!(pool.install(|| build_factor_market_cap_order(&groups, &ids)), expected); + assert_eq!(bits(), before); + } + } + + #[test] + #[should_panic(expected = "factor rows and symbol ids diverged")] + fn compact_market_cap_keys_reject_misaligned_symbol_ids() { + let date = NaiveDate::from_ymd_opt(2024, 1, 2).unwrap(); + build_factor_market_cap_order( + &BTreeMap::from([(date, Vec::new())]), + &BTreeMap::from([(date, vec![1])]), + ); + } + fn market_row(date: &str, prev_close: f64, volume: u64) -> DailyMarketSnapshot { DailyMarketSnapshot { date: NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap(),