diff --git a/crates/fidc-core/src/data.rs b/crates/fidc-core/src/data.rs index 9ae5455..bba85e9 100644 --- a/crates/fidc-core/src/data.rs +++ b/crates/fidc-core/src/data.rs @@ -1335,13 +1335,6 @@ pub struct DataSet { futures_params_by_symbol: Arc>>, } -#[derive(Debug, Clone, Copy)] -pub(crate) struct SymbolSnapshotRefs<'a> { - pub market: Option<&'a DailyMarketSnapshot>, - pub factor: Option<&'a DailyFactorSnapshot>, - pub candidate: Option<&'a CandidateEligibility>, -} - #[derive(Debug, Clone, Copy)] pub(crate) struct StandardRollingMeans { pub close: [Option; 7], @@ -1963,68 +1956,6 @@ impl DataSet { ) } - pub(crate) fn symbol_snapshots_by_id( - &self, - date: NaiveDate, - symbol_id: u32, - ) -> SymbolSnapshotRefs<'_> { - let market_rows = self.market_by_date.get(&date).map(Vec::as_slice); - let market_symbol_ids = self - .market_symbol_ids_by_date - .get(&date) - .map(Vec::as_slice); - let market_index = dense_row_position( - &self.market_row_positions_by_date, - date, - symbol_id, - ) - .or_else(|| { - market_rows - .zip(market_symbol_ids) - .and_then(|(rows, symbol_ids)| symbol_id_index(rows.len(), symbol_ids, symbol_id)) - }); - let market = market_index.and_then(|index| market_rows?.get(index)); - - let factor = self.factor_by_date.get(&date).and_then(|rows| { - dense_row_position(&self.factor_row_positions_by_date, date, symbol_id) - .and_then(|index| rows.get(index)) - .or_else(|| { - self.factor_symbol_ids_by_date - .get(&date) - .and_then(|symbol_ids| { - find_by_symbol_id_with_preferred_index( - rows, - symbol_ids, - symbol_id, - market_index, - ) - }) - }) - }); - let candidate = self.candidate_by_date.get(&date).and_then(|rows| { - dense_row_position(&self.candidate_row_positions_by_date, date, symbol_id) - .and_then(|index| rows.get(index)) - .or_else(|| { - self.candidate_symbol_ids_by_date - .get(&date) - .and_then(|symbol_ids| { - find_by_symbol_id_with_preferred_index( - rows, - symbol_ids, - symbol_id, - market_index, - ) - }) - }) - }); - - SymbolSnapshotRefs { - market, - factor, - candidate, - } - } - pub(crate) fn market_standard_rolling_means_by_symbol_id( &self, date: NaiveDate, @@ -2033,10 +1964,13 @@ impl DataSet { volume_lookbacks: &[usize; 5], include_now: bool, ) -> StandardRollingMeans { + // Both series are built from the same market-date sequence. Reuse the + // indexed boundary lookup instead of repeating it for close and volume. + let series_end = self.market_series_end_index_by_symbol_id(date, symbol_id, include_now); let close = if close_lookbacks.iter().any(|lookback| *lookback > 0) { self.adjusted_close_series_by_symbol_id(symbol_id) .map(|series| { - self.market_series_end_index_by_symbol_id(date, symbol_id, include_now) + series_end .map(|end| series.moving_averages_at_end(end, close_lookbacks)) .unwrap_or_else(|| series.moving_averages(date, close_lookbacks, include_now)) }) @@ -2047,7 +1981,7 @@ impl DataSet { let volume = if volume_lookbacks.iter().any(|lookback| *lookback > 0) { self.market_series_by_symbol_id(symbol_id) .map(|series| { - self.market_series_end_index_by_symbol_id(date, symbol_id, include_now) + series_end .map(|end| series.volume_moving_averages_at_end(end, volume_lookbacks)) .unwrap_or_else(|| { series.volume_moving_averages(date, volume_lookbacks, include_now) @@ -4155,13 +4089,6 @@ fn find_by_symbol_id<'a, T>(rows: &'a [T], symbol_ids: &[u32], symbol_id: u32) - find_by_symbol_id_with_preferred_index(rows, symbol_ids, symbol_id, None) } -fn symbol_id_index(rows_len: usize, symbol_ids: &[u32], symbol_id: u32) -> Option { - if rows_len != symbol_ids.len() { - return None; - } - symbol_ids.binary_search(&symbol_id).ok() -} - fn find_by_symbol_id_with_preferred_index<'a, T>( rows: &'a [T], symbol_ids: &[u32], @@ -4786,7 +4713,7 @@ mod tests { } #[test] - fn combined_symbol_snapshot_lookup_uses_alignment_and_falls_back_for_sparse_rows() { + fn direct_symbol_id_snapshot_lookups_preserve_alignment_for_sparse_rows() { let date = NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap(); let instrument = |symbol: &str| Instrument { symbol: symbol.to_string(), @@ -4848,29 +4775,30 @@ mod tests { for symbol in ["000001.SZ", "600000.SH"] { let symbol_id = data.symbol_id(symbol).unwrap(); - let combined = data.symbol_snapshots_by_id(date, symbol_id); assert_eq!( - combined.market.map(|row| row.symbol.as_str()), data.market_by_symbol_id(date, symbol_id) - .map(|row| row.symbol.as_str()) + .map(|row| row.symbol.as_str()), + Some(symbol) ); assert_eq!( - combined.factor.map(|row| row.symbol.as_str()), data.factor_by_symbol_id(date, symbol_id) - .map(|row| row.symbol.as_str()) + .map(|row| row.symbol.as_str()), + Some(symbol) ); assert_eq!( - combined.candidate.map(|row| row.symbol.as_str()), data.candidate_by_symbol_id(date, symbol_id) - .map(|row| row.symbol.as_str()) + .map(|row| row.symbol.as_str()), + Some(symbol) ); } let signal_id = data.symbol_id("000300.SH").unwrap(); - let signal = data.symbol_snapshots_by_id(date, signal_id); - assert_eq!(signal.market.map(|row| row.symbol.as_str()), Some("000300.SH")); - assert!(signal.factor.is_none()); - assert!(signal.candidate.is_none()); + assert_eq!( + data.market_by_symbol_id(date, signal_id).map(|row| row.symbol.as_str()), + Some("000300.SH") + ); + assert!(data.factor_by_symbol_id(date, signal_id).is_none()); + assert!(data.candidate_by_symbol_id(date, signal_id).is_none()); } #[test] diff --git a/crates/fidc-core/src/platform_expr_strategy.rs b/crates/fidc-core/src/platform_expr_strategy.rs index 7eb9739..9ca70f8 100644 --- a/crates/fidc-core/src/platform_expr_strategy.rs +++ b/crates/fidc-core/src/platform_expr_strategy.rs @@ -990,6 +990,7 @@ pub struct PlatformExprStrategy { stock_rolling_requirements: StockRollingRequirements, stock_extra_factors_required: bool, stock_extra_factor_identifiers: BTreeSet, + stock_extra_factor_map_required: bool, stock_text_factors_required: bool, stock_state_cache_date: RefCell>, stock_state_cache: RefCell< @@ -1267,6 +1268,7 @@ impl PlatformExprStrategy { Self::stock_extra_factors_required_for_config(&config, &prelude_declared_identifiers); let stock_extra_factor_identifiers = Self::stock_extra_factor_identifiers_for_config(&config, &prelude_declared_identifiers); + let stock_extra_factor_map_required = Self::stock_extra_factor_map_required_for_config(&config); let stock_text_factors_required = Self::stock_text_factors_required_for_config( &config, &normalized_stock_filter_expr, @@ -1304,6 +1306,7 @@ impl PlatformExprStrategy { stock_rolling_requirements, stock_extra_factors_required, stock_extra_factor_identifiers, + stock_extra_factor_map_required, stock_text_factors_required, stock_state_cache_date: RefCell::new(None), stock_state_cache: RefCell::new(AHashMap::new()), @@ -3900,28 +3903,28 @@ impl PlatformExprStrategy { return Ok(Arc::clone(state)); } - let execution_snapshots = ctx.data.symbol_snapshots_by_id(date, symbol_id); - let market = execution_snapshots.market.ok_or_else(|| { + let market = ctx.data.market_by_symbol_id(date, symbol_id).ok_or_else(|| { BacktestError::Data(crate::data::DataSetError::MissingSnapshot { kind: "market", date, symbol: symbol.to_string(), }) })?; - let candidate = execution_snapshots.candidate.ok_or_else(|| { + let candidate = ctx.data.candidate_by_symbol_id(date, symbol_id).ok_or_else(|| { BacktestError::Data(crate::data::DataSetError::MissingSnapshot { kind: "candidate", date, symbol: symbol.to_string(), }) })?; - let factor_snapshots = if factor_date == date { - execution_snapshots + let feature_market = if factor_date == date { + market } else { - ctx.data.symbol_snapshots_by_id(factor_date, symbol_id) + ctx.data + .market_by_symbol_id(factor_date, symbol_id) + .unwrap_or(market) }; - let feature_market = factor_snapshots.market.unwrap_or(market); - let factor = factor_snapshots.factor.ok_or_else(|| { + let factor = ctx.data.factor_by_symbol_id(factor_date, symbol_id).ok_or_else(|| { BacktestError::Data(crate::data::DataSetError::MissingSnapshot { kind: "factor", date: factor_date, @@ -4055,6 +4058,10 @@ impl PlatformExprStrategy { factor .extra_factors .iter() + .filter(|(field, _)| { + self.stock_extra_factor_map_required + || self.stock_extra_factor_identifiers.contains(field.as_ref()) + }) .map(|(field, value)| (field.to_string(), *value)) .collect() } else { @@ -9219,6 +9226,56 @@ impl PlatformExprStrategy { .unwrap_or_else(|| self.field_value(candidate))) } + fn rank_reuses_market_cap_order(&self) -> bool { + self.config.rank_expr.trim().is_empty() + && !self.config.rank_desc + && matches!( + self.config.rank_by.trim(), + "market_cap" | "market_cap_bn" + ) + } + + fn selection_candidate_passes_filters( + &self, + ctx: &StrategyContext<'_>, + date: NaiveDate, + day: &DayExpressionState, + candidate: &EligibleUniverseSnapshot, + stock: &StockExpressionState, + diagnostics: &mut Vec, + ) -> Result { + if !ctx.is_lagged_execution() + && let Some(reason) = self.stock_selection_limit_rejection_reason(stock) + { + if diagnostics.len() < 12 { + diagnostics.push(format!("{} rejected by {}", candidate.symbol, reason)); + } + return Ok(false); + } + if !self.stock_passes_expr(ctx, day, stock)? { + if diagnostics.len() < 12 { + diagnostics.push(format!("{} rejected by stock_expr", candidate.symbol)); + } + return Ok(false); + } + if self.config.stop_take_reference_price_mode + == PlatformStopTakeReferencePriceMode::SignalDayPostAdjustedClose + && ctx + .data + .market_latest_back_adjusted_close(date, &candidate.symbol) + .is_none() + { + if diagnostics.len() < 12 { + diagnostics.push(format!( + "{} rejected by missing signal-day post-adjusted close", + candidate.symbol + )); + } + return Ok(false); + } + Ok(true) + } + fn can_sell_position(&self, ctx: &StrategyContext<'_>, date: NaiveDate, symbol: &str) -> bool { self.can_sell_position_at_time(ctx, date, symbol, None) } @@ -9364,6 +9421,41 @@ impl PlatformExprStrategy { universe_factor_date, 5, ); + + // The universe is already stably ordered by market cap. When the + // strategy asks for that exact ascending order and does not need a + // complete ranking for replacement limiting, select directly from the + // ordered stream instead of materializing a second candidate vector. + if self.rank_reuses_market_cap_order() && self.config.daily_replacement_limit == 0 { + let mut selected = Vec::with_capacity(limit.min(universe.len())); + for candidate in universe { + let stock = self.selection_stock_state_with_factor_date( + ctx, + date, + stock_factor_date, + &candidate.symbol, + )?; + let field_value = self.selection_field_value(&candidate, &stock); + if !field_value.is_finite() || field_value < band_low || field_value > band_high { + continue; + } + if self.selection_candidate_passes_filters( + ctx, + date, + day, + &candidate, + &stock, + &mut diagnostics, + )? { + selected.push(candidate.symbol); + if selected.len() >= limit { + break; + } + } + } + return Ok((selected, diagnostics, risk_decisions)); + } + let mut candidates = Vec::new(); let mut missing_rank_count = 0usize; let mut missing_rank_examples = Vec::new(); @@ -9429,54 +9521,37 @@ impl PlatformExprStrategy { } candidates.push((candidate, stock, rank_value)); } - candidates.sort_by(|lhs, rhs| { - let lhs_value = lhs.2; - let rhs_value = rhs.2; - let ordering = if self.config.rank_desc { - rhs_value - .partial_cmp(&lhs_value) - .unwrap_or(std::cmp::Ordering::Equal) - } else { - lhs_value - .partial_cmp(&rhs_value) - .unwrap_or(std::cmp::Ordering::Equal) - }; - if ordering == std::cmp::Ordering::Equal { - lhs.0.symbol.cmp(&rhs.0.symbol) - } else { - ordering - } - }); + if !self.rank_reuses_market_cap_order() { + candidates.sort_by(|lhs, rhs| { + let lhs_value = lhs.2; + let rhs_value = rhs.2; + let ordering = if self.config.rank_desc { + rhs_value + .partial_cmp(&lhs_value) + .unwrap_or(std::cmp::Ordering::Equal) + } else { + lhs_value + .partial_cmp(&rhs_value) + .unwrap_or(std::cmp::Ordering::Equal) + }; + if ordering == std::cmp::Ordering::Equal { + lhs.0.symbol.cmp(&rhs.0.symbol) + } else { + ordering + } + }); + } let mut selected = Vec::new(); for (candidate, stock, _) in candidates { - if !ctx.is_lagged_execution() - && let Some(reason) = self.stock_selection_limit_rejection_reason(&stock) - { - if diagnostics.len() < 12 { - diagnostics.push(format!("{} rejected by {}", candidate.symbol, reason)); - } - continue; - } - if !self.stock_passes_expr(ctx, day, &stock)? { - if diagnostics.len() < 12 { - diagnostics.push(format!("{} rejected by stock_expr", candidate.symbol)); - } - continue; - } - if self.config.stop_take_reference_price_mode - == PlatformStopTakeReferencePriceMode::SignalDayPostAdjustedClose - && ctx - .data - .market_latest_back_adjusted_close(date, &candidate.symbol) - .is_none() - { - if diagnostics.len() < 12 { - diagnostics.push(format!( - "{} rejected by missing signal-day post-adjusted close", - candidate.symbol - )); - } + if !self.selection_candidate_passes_filters( + ctx, + date, + day, + &candidate, + &stock, + &mut diagnostics, + )? { continue; } selected.push(candidate.symbol.clone()); @@ -9597,6 +9672,30 @@ impl PlatformExprStrategy { .any(|expr| Self::expr_requires_stock_extra_factors(expr, prelude_declared_identifiers)) } + fn stock_extra_factor_map_required_for_config(config: &PlatformExprStrategyConfig) -> bool { + // A dynamic factors/factor map is part of the expression contract: its + // caller may access any published field by key. Keep the full map for + // that case and for explicit actions whose predicates are evaluated at + // runtime; direct field references use the projected identifier set. + if Self::has_stock_explicit_actions(config) { + return true; + } + [ + config.prelude.as_str(), + config.stock_filter_expr.as_str(), + config.buy_scale_expr.as_str(), + config.stop_loss_expr.as_str(), + config.take_profit_expr.as_str(), + config.rank_expr.as_str(), + config.market_cap_field.as_str(), + config.rank_by.as_str(), + ] + .into_iter() + .map(Self::normalize_expr) + .map(|expr| Self::extract_identifier_candidates(&expr)) + .any(|identifiers| identifiers.contains("factors") || identifiers.contains("factor")) + } + fn stock_extra_factor_identifiers_for_config( config: &PlatformExprStrategyConfig, prelude_declared_identifiers: &BTreeSet, @@ -13749,6 +13848,28 @@ mod tests { ); } + #[test] + fn market_cap_ascending_rank_reuses_only_equivalent_universe_order() { + let mut cfg = PlatformExprStrategyConfig::microcap_rotation(); + cfg.rank_by = "market_cap".to_string(); + let strategy = PlatformExprStrategy::new(cfg.clone()); + assert!(strategy.rank_reuses_market_cap_order()); + + cfg.rank_by = "market_cap_bn".to_string(); + assert!(PlatformExprStrategy::new(cfg.clone()).rank_reuses_market_cap_order()); + + cfg.rank_desc = true; + assert!(!PlatformExprStrategy::new(cfg.clone()).rank_reuses_market_cap_order()); + + cfg.rank_desc = false; + cfg.rank_expr = "close".to_string(); + assert!(!PlatformExprStrategy::new(cfg.clone()).rank_reuses_market_cap_order()); + + cfg.rank_expr.clear(); + cfg.rank_by = "free_float_cap".to_string(); + assert!(!PlatformExprStrategy::new(cfg).rank_reuses_market_cap_order()); + } + #[test] fn current_rolling_helpers_do_not_load_factor_maps_or_decision_rollings() { let mut cfg = PlatformExprStrategyConfig::microcap_rotation(); @@ -13831,7 +13952,10 @@ mod tests { pe_ttm: 8.0, turnover_ratio: Some(1.0), effective_turnover_ratio: Some(1.0), - extra_factors: BTreeMap::from([("model_score".into(), 2.0)]), + extra_factors: BTreeMap::from([ + ("model_score".into(), 2.0), + ("unused_factor".into(), 9.0), + ]), }, ], symbols @@ -13901,6 +14025,25 @@ mod tests { .stock_passes_expr(&ctx, &day, &present_stock) .expect("present stock filter") ); + assert!(!strategy.stock_extra_factor_map_required); + assert!(present_stock.extra_factors.contains_key("model_score")); + assert!(!present_stock.extra_factors.contains_key("unused_factor")); + + let mut map_cfg = PlatformExprStrategyConfig::microcap_rotation(); + map_cfg.signal_symbol = present_symbol.to_string(); + map_cfg.stock_filter_expr = "factors[\"unused_factor\"] > 0".to_string(); + let map_strategy = PlatformExprStrategy::new(map_cfg); + assert!(map_strategy.stock_extra_factor_map_required); + let map_stock = map_strategy + .stock_state_with_factor_date(&ctx, date, date, present_symbol) + .expect("factor map stock state"); + assert!(map_stock.extra_factors.contains_key("unused_factor")); + let map_day = map_strategy.day_state(&ctx, date).expect("factor map day state"); + assert!( + map_strategy + .stock_passes_expr(&ctx, &map_day, &map_stock) + .expect("factor map filter") + ); } #[test]