diff --git a/crates/fidc-core/src/data.rs b/crates/fidc-core/src/data.rs index 3618cbf..388b82e 100644 --- a/crates/fidc-core/src/data.rs +++ b/crates/fidc-core/src/data.rs @@ -1116,10 +1116,13 @@ pub struct DataSet { instruments: HashMap, calendar: TradingCalendar, market_by_date: BTreeMap>>, + market_symbol_ids_by_date: Arc>>, factor_by_date: BTreeMap>>, + factor_symbol_ids_by_date: Arc>>, factor_text_by_date: BTreeMap>, factor_text_index: HashMap<(NaiveDate, String, String), FactorTextValue>, candidate_by_date: BTreeMap>>, + candidate_symbol_ids_by_date: Arc>>, corporate_actions_by_date: BTreeMap>, execution_quotes_by_date: HashMap>>, order_book_depth_index: HashMap<(NaiveDate, String), Vec>, @@ -1127,6 +1130,7 @@ pub struct DataSet { market_series_by_symbol: Arc>>, adjusted_close_series_by_symbol: Arc>>, benchmark_series_cache: BenchmarkPriceSeries, + symbol_id_by_code: Arc>, eligible_universe_by_date: Arc>>>, benchmark_code: String, futures_params_by_symbol: HashMap>, @@ -1330,6 +1334,24 @@ impl DataSet { let mut candidate_by_date = group_arc_by_date(&candidates, |item| item.date); sort_arc_groups_by_symbol(&mut candidate_by_date, |item| item.symbol.as_str()); + let symbol_id_by_code = build_symbol_id_index( + &instruments, + &market_by_date, + &factor_by_date, + &candidate_by_date, + ); + let market_symbol_ids_by_date = + build_group_symbol_ids(&market_by_date, &symbol_id_by_code, |item| { + item.symbol.as_str() + }); + let factor_symbol_ids_by_date = + build_group_symbol_ids(&factor_by_date, &symbol_id_by_code, |item| { + item.symbol.as_str() + }); + let candidate_symbol_ids_by_date = + build_group_symbol_ids(&candidate_by_date, &symbol_id_by_code, |item| { + item.symbol.as_str() + }); let corporate_actions_by_date = group_by_date(corporate_actions, |item| item.date); let execution_quotes_by_date = build_execution_quote_index(execution_quotes); let order_book_depth_index = build_order_book_depth_index(order_book_depth); @@ -1346,10 +1368,13 @@ impl DataSet { instruments, calendar, market_by_date, + market_symbol_ids_by_date: Arc::new(market_symbol_ids_by_date), factor_by_date, + factor_symbol_ids_by_date: Arc::new(factor_symbol_ids_by_date), factor_text_by_date, factor_text_index, candidate_by_date, + candidate_symbol_ids_by_date: Arc::new(candidate_symbol_ids_by_date), corporate_actions_by_date, execution_quotes_by_date, order_book_depth_index, @@ -1357,6 +1382,7 @@ impl DataSet { market_series_by_symbol: Arc::new(market_series_by_symbol), adjusted_close_series_by_symbol: Arc::new(adjusted_close_series_by_symbol), benchmark_series_cache, + symbol_id_by_code: Arc::new(symbol_id_by_code), eligible_universe_by_date: Arc::new(OnceLock::new()), benchmark_code, futures_params_by_symbol, @@ -1401,9 +1427,12 @@ impl DataSet { } pub fn market(&self, date: NaiveDate, symbol: &str) -> Option<&DailyMarketSnapshot> { - self.market_by_date - .get(&date) - .and_then(|rows| find_arc_by_symbol(rows, symbol, |row| row.symbol.as_str())) + let symbol_id = *self.symbol_id_by_code.get(symbol)?; + find_arc_by_symbol_id( + self.market_by_date.get(&date)?, + self.market_symbol_ids_by_date.get(&date)?, + symbol_id, + ) } fn market_series(&self, symbol: &str) -> Option<&SymbolPriceSeries> { @@ -1417,15 +1446,21 @@ impl DataSet { } pub fn factor(&self, date: NaiveDate, symbol: &str) -> Option<&DailyFactorSnapshot> { - self.factor_by_date - .get(&date) - .and_then(|rows| find_arc_by_symbol(rows, symbol, |row| row.symbol.as_str())) + let symbol_id = *self.symbol_id_by_code.get(symbol)?; + find_arc_by_symbol_id( + self.factor_by_date.get(&date)?, + self.factor_symbol_ids_by_date.get(&date)?, + symbol_id, + ) } pub fn candidate(&self, date: NaiveDate, symbol: &str) -> Option<&CandidateEligibility> { - self.candidate_by_date - .get(&date) - .and_then(|rows| find_arc_by_symbol(rows, symbol, |row| row.symbol.as_str())) + let symbol_id = *self.symbol_id_by_code.get(symbol)?; + find_arc_by_symbol_id( + self.candidate_by_date.get(&date)?, + self.candidate_symbol_ids_by_date.get(&date)?, + symbol_id, + ) } pub fn benchmark(&self, date: NaiveDate) -> Option<&BenchmarkSnapshot> { @@ -3079,6 +3114,88 @@ where } } +fn build_symbol_id_index( + instruments: &HashMap, + market_by_date: &BTreeMap>>, + factor_by_date: &BTreeMap>>, + candidate_by_date: &BTreeMap>>, +) -> HashMap { + let mut symbols = instruments.keys().cloned().collect::>(); + for rows in market_by_date.values() { + for row in rows { + if !symbols.contains(row.symbol.as_str()) { + symbols.insert(row.symbol.clone()); + } + } + } + for rows in factor_by_date.values() { + for row in rows { + if !symbols.contains(row.symbol.as_str()) { + symbols.insert(row.symbol.clone()); + } + } + } + for rows in candidate_by_date.values() { + for row in rows { + if !symbols.contains(row.symbol.as_str()) { + symbols.insert(row.symbol.clone()); + } + } + } + let mut symbols = symbols.into_iter().collect::>(); + symbols.sort_unstable(); + symbols + .into_iter() + .enumerate() + .map(|(index, symbol)| { + ( + symbol, + u32::try_from(index).expect("FIDC symbol index exceeds u32 capacity"), + ) + }) + .collect() +} + +fn build_group_symbol_ids( + groups: &BTreeMap>>, + symbol_id_by_code: &HashMap, + symbol_of: F, +) -> BTreeMap> +where + F: Fn(&T) -> &str + Copy, +{ + groups + .iter() + .map(|(date, rows)| { + let symbol_ids = rows + .iter() + .map(|row| { + *symbol_id_by_code + .get(symbol_of(row.as_ref())) + .expect("snapshot symbol missing from FIDC symbol index") + }) + .collect::>(); + debug_assert!(symbol_ids.windows(2).all(|window| window[0] < window[1])); + (*date, symbol_ids) + }) + .collect() +} + +fn find_arc_by_symbol_id<'a, T>( + rows: &'a [Arc], + symbol_ids: &[u32], + symbol_id: u32, +) -> Option<&'a T> { + if rows.len() != symbol_ids.len() { + return None; + } + symbol_ids + .binary_search(&symbol_id) + .ok() + .and_then(|index| rows.get(index)) + .map(Arc::as_ref) +} + fn find_arc_by_symbol<'a, T, F>(rows: &'a [Arc], symbol: &str, symbol_of: F) -> Option<&'a T> where F: Fn(&T) -> &str,