From c46dcf817b660b719161bbc4275159c37900a17c Mon Sep 17 00:00:00 2001 From: boris Date: Fri, 28 Aug 2026 07:11:19 +0800 Subject: [PATCH] perf(core): build sorted dataset components linearly --- crates/fidc-core/src/data.rs | 308 ++++++++++++++++++++++++++++++++++- 1 file changed, 302 insertions(+), 6 deletions(-) diff --git a/crates/fidc-core/src/data.rs b/crates/fidc-core/src/data.rs index e502d3a..ede6ccc 100644 --- a/crates/fidc-core/src/data.rs +++ b/crates/fidc-core/src/data.rs @@ -72,6 +72,12 @@ pub enum DataSetError { }, #[error("benchmark snapshot missing for {date}")] MissingBenchmark { date: NaiveDate }, + #[error("{kind} snapshots are not strictly sorted by date and symbol at {date} / {symbol}")] + InvalidComponentOrder { + kind: &'static str, + date: NaiveDate, + symbol: String, + }, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1308,6 +1314,30 @@ impl DataSet { ) } + pub fn from_date_symbol_sorted_components_with_actions_and_quotes( + instruments: Vec, + market: Vec, + factors: Vec, + candidates: Vec, + benchmarks: Vec, + corporate_actions: Vec, + execution_quotes: Vec, + ) -> Result { + Self::from_components_with_actions_quotes_futures_depth_and_factor_texts_ordered( + instruments, + market, + factors, + candidates, + benchmarks, + corporate_actions, + execution_quotes, + Vec::new(), + Vec::new(), + Vec::new(), + true, + ) + } + pub fn from_components_with_actions_quotes_and_futures( instruments: Vec, market: Vec, @@ -1389,6 +1419,35 @@ impl DataSet { futures_params: Vec, order_book_depth: Vec, factor_texts: Vec, + ) -> Result { + Self::from_components_with_actions_quotes_futures_depth_and_factor_texts_ordered( + instruments, + market, + factors, + candidates, + benchmarks, + corporate_actions, + execution_quotes, + futures_params, + order_book_depth, + factor_texts, + false, + ) + } + + #[allow(clippy::too_many_arguments)] + fn from_components_with_actions_quotes_futures_depth_and_factor_texts_ordered( + instruments: Vec, + market: Vec, + factors: Vec, + candidates: Vec, + benchmarks: Vec, + corporate_actions: Vec, + execution_quotes: Vec, + futures_params: Vec, + order_book_depth: Vec, + factor_texts: Vec, + date_symbol_sorted: bool, ) -> Result { let benchmark_code = collect_benchmark_code(&benchmarks)?; let calendar = TradingCalendar::new(benchmarks.iter().map(|item| item.date).collect()); @@ -1399,11 +1458,31 @@ impl DataSet { .map(|instrument| (instrument.symbol.clone(), instrument)) .collect::>(); - let mut market_by_date = group_by_date(market, |item| item.date); - sort_groups_by_symbol(&mut market_by_date, |item| item.symbol.as_str()); + let market_by_date = if date_symbol_sorted { + group_date_symbol_sorted( + market, + |item| item.date, + |item| item.symbol.as_str(), + "market", + )? + } else { + let mut grouped = group_by_date(market, |item| item.date); + sort_groups_by_symbol(&mut grouped, |item| item.symbol.as_str()); + grouped + }; - let mut factor_by_date = group_by_date(factors, |item| item.date); - sort_groups_by_symbol(&mut factor_by_date, |item| item.symbol.as_str()); + let factor_by_date = if date_symbol_sorted { + group_date_symbol_sorted( + factors, + |item| item.date, + |item| item.symbol.as_str(), + "factor", + )? + } else { + let mut grouped = group_by_date(factors, |item| item.date); + sort_groups_by_symbol(&mut grouped, |item| item.symbol.as_str()); + grouped + }; let mut market_rows_by_symbol = AHashMap::>::new(); for row in market_by_date.values().flatten() { if let Some(rows) = market_rows_by_symbol.get_mut(row.symbol.as_str()) { @@ -1459,8 +1538,18 @@ impl DataSet { .map(|item| ((item.date, item.symbol.clone(), item.field.clone()), item)) .collect::>(); - let mut candidate_by_date = group_by_date(candidates, |item| item.date); - sort_groups_by_symbol(&mut candidate_by_date, |item| item.symbol.as_str()); + let candidate_by_date = if date_symbol_sorted { + group_date_symbol_sorted( + candidates, + |item| item.date, + |item| item.symbol.as_str(), + "candidate", + )? + } else { + let mut grouped = group_by_date(candidates, |item| item.date); + sort_groups_by_symbol(&mut grouped, |item| item.symbol.as_str()); + grouped + }; let symbol_id_by_code = build_symbol_id_index( &instruments, &market_by_date, @@ -3477,6 +3566,58 @@ where grouped } +fn group_date_symbol_sorted( + rows: Vec, + date_of: D, + symbol_of: S, + kind: &'static str, +) -> Result>, DataSetError> +where + D: Fn(&T) -> NaiveDate + Copy, + S: Fn(&T) -> &str + Copy, +{ + let mut grouped = BTreeMap::>::new(); + let mut rows = rows.into_iter().peekable(); + while let Some(first) = rows.next() { + let date = date_of(&first); + if grouped + .last_key_value() + .is_some_and(|(previous, _)| *previous >= date) + { + return Err(DataSetError::InvalidComponentOrder { + kind, + date, + symbol: symbol_of(&first).to_string(), + }); + } + let mut day_rows = vec![first]; + while rows.peek().is_some_and(|row| date_of(row) == date) { + day_rows.push(rows.next().expect("peeked component row")); + } + if let Some(next) = rows.peek() + && date_of(next) < date + { + return Err(DataSetError::InvalidComponentOrder { + kind, + date: date_of(next), + symbol: symbol_of(next).to_string(), + }); + } + if let Some(window) = day_rows + .windows(2) + .find(|window| symbol_of(&window[0]) >= symbol_of(&window[1])) + { + return Err(DataSetError::InvalidComponentOrder { + kind, + date, + symbol: symbol_of(&window[1]).to_string(), + }); + } + grouped.insert(date, day_rows); + } + Ok(grouped) +} + fn sort_groups_by_symbol(groups: &mut BTreeMap>, symbol_of: F) where F: Fn(&T) -> &str + Copy, @@ -3891,6 +4032,161 @@ mod tests { } } + fn instrument(symbol: &str) -> Instrument { + Instrument { + symbol: symbol.to_string(), + name: symbol.to_string(), + board: symbol + .rsplit_once('.') + .map(|(_, board)| board) + .unwrap_or("") + .to_string(), + round_lot: 100, + listed_at: None, + delisted_at: None, + status: "active".to_string(), + } + } + + fn market_row_for(date: &str, symbol: &str, close: f64) -> DailyMarketSnapshot { + let mut row = market_row(date, close, 1_000_000); + row.symbol = symbol.to_string(); + row + } + + fn factor_row_for(date: &str, symbol: &str, market_cap_bn: f64) -> DailyFactorSnapshot { + DailyFactorSnapshot { + date: NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap(), + symbol: symbol.to_string(), + market_cap_bn, + free_float_cap_bn: market_cap_bn * 0.8, + pe_ttm: 0.0, + turnover_ratio: Some(0.02), + effective_turnover_ratio: Some(0.01), + extra_factors: NumericFactorMap::from([(Cow::Borrowed("quality"), 1.0)]), + } + } + + fn candidate_row_for(date: &str, symbol: &str) -> CandidateEligibility { + CandidateEligibility { + date: NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap(), + symbol: symbol.to_string(), + is_st: false, + is_star_st: false, + is_new_listing: false, + is_paused: false, + allow_buy: true, + allow_sell: true, + is_kcb: false, + is_one_yuan: false, + risk_level_code: None, + } + } + + #[test] + fn sorted_component_constructor_matches_generic_constructor() { + let instruments = vec![instrument("000001.SZ"), instrument("600000.SH")]; + let market = vec![ + market_row_for("2025-01-02", "000001.SZ", 10.0), + market_row_for("2025-01-02", "600000.SH", 12.0), + market_row_for("2025-01-03", "000001.SZ", 10.2), + market_row_for("2025-01-03", "600000.SH", 12.2), + ]; + let factors = vec![ + factor_row_for("2025-01-02", "000001.SZ", 100.0), + factor_row_for("2025-01-02", "600000.SH", 120.0), + factor_row_for("2025-01-03", "000001.SZ", 102.0), + factor_row_for("2025-01-03", "600000.SH", 122.0), + ]; + let candidates = vec![ + candidate_row_for("2025-01-02", "000001.SZ"), + candidate_row_for("2025-01-02", "600000.SH"), + candidate_row_for("2025-01-03", "000001.SZ"), + candidate_row_for("2025-01-03", "600000.SH"), + ]; + let benchmarks = vec![ + benchmark_row("2025-01-02", 20.0), + benchmark_row("2025-01-03", 20.2), + ]; + let generic = DataSet::from_components( + instruments.clone(), + market.clone(), + factors.clone(), + candidates.clone(), + benchmarks.clone(), + ) + .unwrap(); + let sorted = DataSet::from_date_symbol_sorted_components_with_actions_and_quotes( + instruments, + market, + factors, + candidates, + benchmarks, + Vec::new(), + Vec::new(), + ) + .unwrap(); + + for date in ["2025-01-02", "2025-01-03"] { + let date = NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap(); + for symbol in ["000001.SZ", "600000.SH"] { + assert_eq!( + generic.market(date, symbol).unwrap().close, + sorted.market(date, symbol).unwrap().close + ); + assert_eq!( + generic.factor(date, symbol).unwrap().market_cap_bn, + sorted.factor(date, symbol).unwrap().market_cap_bn + ); + assert_eq!( + generic.candidate(date, symbol).unwrap().allow_buy, + sorted.candidate(date, symbol).unwrap().allow_buy + ); + } + } + } + + #[test] + fn sorted_component_constructor_rejects_unsorted_or_duplicate_rows() { + let benchmarks = vec![ + benchmark_row("2025-01-02", 20.0), + benchmark_row("2025-01-03", 20.2), + ]; + let unsorted = DataSet::from_date_symbol_sorted_components_with_actions_and_quotes( + vec![instrument("000001.SZ")], + vec![ + market_row_for("2025-01-03", "000001.SZ", 10.2), + market_row_for("2025-01-02", "000001.SZ", 10.0), + ], + Vec::new(), + Vec::new(), + benchmarks.clone(), + Vec::new(), + Vec::new(), + ); + assert!(matches!( + unsorted, + Err(DataSetError::InvalidComponentOrder { kind: "market", .. }) + )); + + let duplicate = DataSet::from_date_symbol_sorted_components_with_actions_and_quotes( + vec![instrument("000001.SZ")], + vec![ + market_row_for("2025-01-02", "000001.SZ", 10.0), + market_row_for("2025-01-02", "000001.SZ", 10.1), + ], + Vec::new(), + Vec::new(), + benchmarks, + Vec::new(), + Vec::new(), + ); + assert!(matches!( + duplicate, + Err(DataSetError::InvalidComponentOrder { kind: "market", .. }) + )); + } + #[test] fn dataset_clone_shares_immutable_base_and_isolates_execution_quotes() { let date = NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap();