From 87c18574a87441f82f33951bc21e9b88ebea4397 Mon Sep 17 00:00:00 2001 From: boris Date: Fri, 28 Aug 2026 07:20:20 +0800 Subject: [PATCH] Revert "perf(core): build sorted dataset components linearly" This reverts commit c46dcf817b660b719161bbc4275159c37900a17c. --- crates/fidc-core/src/data.rs | 308 +---------------------------------- 1 file changed, 6 insertions(+), 302 deletions(-) diff --git a/crates/fidc-core/src/data.rs b/crates/fidc-core/src/data.rs index ede6ccc..e502d3a 100644 --- a/crates/fidc-core/src/data.rs +++ b/crates/fidc-core/src/data.rs @@ -72,12 +72,6 @@ pub enum DataSetError { }, #[error("benchmark snapshot missing for {date}")] MissingBenchmark { date: NaiveDate }, - #[error("{kind} snapshots are not strictly sorted by date and symbol at {date} / {symbol}")] - InvalidComponentOrder { - kind: &'static str, - date: NaiveDate, - symbol: String, - }, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1314,30 +1308,6 @@ impl DataSet { ) } - pub fn from_date_symbol_sorted_components_with_actions_and_quotes( - instruments: Vec, - market: Vec, - factors: Vec, - candidates: Vec, - benchmarks: Vec, - corporate_actions: Vec, - execution_quotes: Vec, - ) -> Result { - Self::from_components_with_actions_quotes_futures_depth_and_factor_texts_ordered( - instruments, - market, - factors, - candidates, - benchmarks, - corporate_actions, - execution_quotes, - Vec::new(), - Vec::new(), - Vec::new(), - true, - ) - } - pub fn from_components_with_actions_quotes_and_futures( instruments: Vec, market: Vec, @@ -1419,35 +1389,6 @@ impl DataSet { futures_params: Vec, order_book_depth: Vec, factor_texts: Vec, - ) -> Result { - Self::from_components_with_actions_quotes_futures_depth_and_factor_texts_ordered( - instruments, - market, - factors, - candidates, - benchmarks, - corporate_actions, - execution_quotes, - futures_params, - order_book_depth, - factor_texts, - false, - ) - } - - #[allow(clippy::too_many_arguments)] - fn from_components_with_actions_quotes_futures_depth_and_factor_texts_ordered( - instruments: Vec, - market: Vec, - factors: Vec, - candidates: Vec, - benchmarks: Vec, - corporate_actions: Vec, - execution_quotes: Vec, - futures_params: Vec, - order_book_depth: Vec, - factor_texts: Vec, - date_symbol_sorted: bool, ) -> Result { let benchmark_code = collect_benchmark_code(&benchmarks)?; let calendar = TradingCalendar::new(benchmarks.iter().map(|item| item.date).collect()); @@ -1458,31 +1399,11 @@ impl DataSet { .map(|instrument| (instrument.symbol.clone(), instrument)) .collect::>(); - let market_by_date = if date_symbol_sorted { - group_date_symbol_sorted( - market, - |item| item.date, - |item| item.symbol.as_str(), - "market", - )? - } else { - let mut grouped = group_by_date(market, |item| item.date); - sort_groups_by_symbol(&mut grouped, |item| item.symbol.as_str()); - grouped - }; + let mut market_by_date = group_by_date(market, |item| item.date); + sort_groups_by_symbol(&mut market_by_date, |item| item.symbol.as_str()); - let factor_by_date = if date_symbol_sorted { - group_date_symbol_sorted( - factors, - |item| item.date, - |item| item.symbol.as_str(), - "factor", - )? - } else { - let mut grouped = group_by_date(factors, |item| item.date); - sort_groups_by_symbol(&mut grouped, |item| item.symbol.as_str()); - grouped - }; + let mut factor_by_date = group_by_date(factors, |item| item.date); + sort_groups_by_symbol(&mut factor_by_date, |item| item.symbol.as_str()); let mut market_rows_by_symbol = AHashMap::>::new(); for row in market_by_date.values().flatten() { if let Some(rows) = market_rows_by_symbol.get_mut(row.symbol.as_str()) { @@ -1538,18 +1459,8 @@ impl DataSet { .map(|item| ((item.date, item.symbol.clone(), item.field.clone()), item)) .collect::>(); - let candidate_by_date = if date_symbol_sorted { - group_date_symbol_sorted( - candidates, - |item| item.date, - |item| item.symbol.as_str(), - "candidate", - )? - } else { - let mut grouped = group_by_date(candidates, |item| item.date); - sort_groups_by_symbol(&mut grouped, |item| item.symbol.as_str()); - grouped - }; + let mut candidate_by_date = group_by_date(candidates, |item| item.date); + sort_groups_by_symbol(&mut candidate_by_date, |item| item.symbol.as_str()); let symbol_id_by_code = build_symbol_id_index( &instruments, &market_by_date, @@ -3566,58 +3477,6 @@ where grouped } -fn group_date_symbol_sorted( - rows: Vec, - date_of: D, - symbol_of: S, - kind: &'static str, -) -> Result>, DataSetError> -where - D: Fn(&T) -> NaiveDate + Copy, - S: Fn(&T) -> &str + Copy, -{ - let mut grouped = BTreeMap::>::new(); - let mut rows = rows.into_iter().peekable(); - while let Some(first) = rows.next() { - let date = date_of(&first); - if grouped - .last_key_value() - .is_some_and(|(previous, _)| *previous >= date) - { - return Err(DataSetError::InvalidComponentOrder { - kind, - date, - symbol: symbol_of(&first).to_string(), - }); - } - let mut day_rows = vec![first]; - while rows.peek().is_some_and(|row| date_of(row) == date) { - day_rows.push(rows.next().expect("peeked component row")); - } - if let Some(next) = rows.peek() - && date_of(next) < date - { - return Err(DataSetError::InvalidComponentOrder { - kind, - date: date_of(next), - symbol: symbol_of(next).to_string(), - }); - } - if let Some(window) = day_rows - .windows(2) - .find(|window| symbol_of(&window[0]) >= symbol_of(&window[1])) - { - return Err(DataSetError::InvalidComponentOrder { - kind, - date, - symbol: symbol_of(&window[1]).to_string(), - }); - } - grouped.insert(date, day_rows); - } - Ok(grouped) -} - fn sort_groups_by_symbol(groups: &mut BTreeMap>, symbol_of: F) where F: Fn(&T) -> &str + Copy, @@ -4032,161 +3891,6 @@ mod tests { } } - fn instrument(symbol: &str) -> Instrument { - Instrument { - symbol: symbol.to_string(), - name: symbol.to_string(), - board: symbol - .rsplit_once('.') - .map(|(_, board)| board) - .unwrap_or("") - .to_string(), - round_lot: 100, - listed_at: None, - delisted_at: None, - status: "active".to_string(), - } - } - - fn market_row_for(date: &str, symbol: &str, close: f64) -> DailyMarketSnapshot { - let mut row = market_row(date, close, 1_000_000); - row.symbol = symbol.to_string(); - row - } - - fn factor_row_for(date: &str, symbol: &str, market_cap_bn: f64) -> DailyFactorSnapshot { - DailyFactorSnapshot { - date: NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap(), - symbol: symbol.to_string(), - market_cap_bn, - free_float_cap_bn: market_cap_bn * 0.8, - pe_ttm: 0.0, - turnover_ratio: Some(0.02), - effective_turnover_ratio: Some(0.01), - extra_factors: NumericFactorMap::from([(Cow::Borrowed("quality"), 1.0)]), - } - } - - fn candidate_row_for(date: &str, symbol: &str) -> CandidateEligibility { - CandidateEligibility { - date: NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap(), - symbol: symbol.to_string(), - is_st: false, - is_star_st: false, - is_new_listing: false, - is_paused: false, - allow_buy: true, - allow_sell: true, - is_kcb: false, - is_one_yuan: false, - risk_level_code: None, - } - } - - #[test] - fn sorted_component_constructor_matches_generic_constructor() { - let instruments = vec![instrument("000001.SZ"), instrument("600000.SH")]; - let market = vec![ - market_row_for("2025-01-02", "000001.SZ", 10.0), - market_row_for("2025-01-02", "600000.SH", 12.0), - market_row_for("2025-01-03", "000001.SZ", 10.2), - market_row_for("2025-01-03", "600000.SH", 12.2), - ]; - let factors = vec![ - factor_row_for("2025-01-02", "000001.SZ", 100.0), - factor_row_for("2025-01-02", "600000.SH", 120.0), - factor_row_for("2025-01-03", "000001.SZ", 102.0), - factor_row_for("2025-01-03", "600000.SH", 122.0), - ]; - let candidates = vec![ - candidate_row_for("2025-01-02", "000001.SZ"), - candidate_row_for("2025-01-02", "600000.SH"), - candidate_row_for("2025-01-03", "000001.SZ"), - candidate_row_for("2025-01-03", "600000.SH"), - ]; - let benchmarks = vec![ - benchmark_row("2025-01-02", 20.0), - benchmark_row("2025-01-03", 20.2), - ]; - let generic = DataSet::from_components( - instruments.clone(), - market.clone(), - factors.clone(), - candidates.clone(), - benchmarks.clone(), - ) - .unwrap(); - let sorted = DataSet::from_date_symbol_sorted_components_with_actions_and_quotes( - instruments, - market, - factors, - candidates, - benchmarks, - Vec::new(), - Vec::new(), - ) - .unwrap(); - - for date in ["2025-01-02", "2025-01-03"] { - let date = NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap(); - for symbol in ["000001.SZ", "600000.SH"] { - assert_eq!( - generic.market(date, symbol).unwrap().close, - sorted.market(date, symbol).unwrap().close - ); - assert_eq!( - generic.factor(date, symbol).unwrap().market_cap_bn, - sorted.factor(date, symbol).unwrap().market_cap_bn - ); - assert_eq!( - generic.candidate(date, symbol).unwrap().allow_buy, - sorted.candidate(date, symbol).unwrap().allow_buy - ); - } - } - } - - #[test] - fn sorted_component_constructor_rejects_unsorted_or_duplicate_rows() { - let benchmarks = vec![ - benchmark_row("2025-01-02", 20.0), - benchmark_row("2025-01-03", 20.2), - ]; - let unsorted = DataSet::from_date_symbol_sorted_components_with_actions_and_quotes( - vec![instrument("000001.SZ")], - vec![ - market_row_for("2025-01-03", "000001.SZ", 10.2), - market_row_for("2025-01-02", "000001.SZ", 10.0), - ], - Vec::new(), - Vec::new(), - benchmarks.clone(), - Vec::new(), - Vec::new(), - ); - assert!(matches!( - unsorted, - Err(DataSetError::InvalidComponentOrder { kind: "market", .. }) - )); - - let duplicate = DataSet::from_date_symbol_sorted_components_with_actions_and_quotes( - vec![instrument("000001.SZ")], - vec![ - market_row_for("2025-01-02", "000001.SZ", 10.0), - market_row_for("2025-01-02", "000001.SZ", 10.1), - ], - Vec::new(), - Vec::new(), - benchmarks, - Vec::new(), - Vec::new(), - ); - assert!(matches!( - duplicate, - Err(DataSetError::InvalidComponentOrder { kind: "market", .. }) - )); - } - #[test] fn dataset_clone_shares_immutable_base_and_isolates_execution_quotes() { let date = NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap();