Merge remote-tracking branch 'origin/main'

crates/fidc-core/src/data.rs

@@ -1335,13 +1335,6 @@ pub struct DataSet {
     futures_params_by_symbol: Arc<HashMap<String, Vec<FuturesTradingParameter>>>,
 }
 
-#[derive(Debug, Clone, Copy)]
-pub(crate) struct SymbolSnapshotRefs<'a> {
-    pub market: Option<&'a DailyMarketSnapshot>,
-    pub factor: Option<&'a DailyFactorSnapshot>,
-    pub candidate: Option<&'a CandidateEligibility>,
-}
-
 #[derive(Debug, Clone, Copy)]
 pub(crate) struct StandardRollingMeans {
     pub close: [Option<f64>; 7],
@@ -1963,68 +1956,6 @@ impl DataSet {
         )
     }
 
-    pub(crate) fn symbol_snapshots_by_id(
-        &self,
-        date: NaiveDate,
-        symbol_id: u32,
-    ) -> SymbolSnapshotRefs<'_> {
-        let market_rows = self.market_by_date.get(&date).map(Vec::as_slice);
-        let market_symbol_ids = self
-            .market_symbol_ids_by_date
-            .get(&date)
-            .map(Vec::as_slice);
-        let market_index = dense_row_position(
-            &self.market_row_positions_by_date,
-            date,
-            symbol_id,
-        )
-        .or_else(|| {
-            market_rows
-                .zip(market_symbol_ids)
-                .and_then(|(rows, symbol_ids)| symbol_id_index(rows.len(), symbol_ids, symbol_id))
-        });
-        let market = market_index.and_then(|index| market_rows?.get(index));
-
-        let factor = self.factor_by_date.get(&date).and_then(|rows| {
-            dense_row_position(&self.factor_row_positions_by_date, date, symbol_id)
-                .and_then(|index| rows.get(index))
-                .or_else(|| {
-                    self.factor_symbol_ids_by_date
-                        .get(&date)
-                        .and_then(|symbol_ids| {
-                            find_by_symbol_id_with_preferred_index(
-                                rows,
-                                symbol_ids,
-                                symbol_id,
-                                market_index,
-                            )
-                        })
-                })
-        });
-        let candidate = self.candidate_by_date.get(&date).and_then(|rows| {
-            dense_row_position(&self.candidate_row_positions_by_date, date, symbol_id)
-                .and_then(|index| rows.get(index))
-                .or_else(|| {
-                    self.candidate_symbol_ids_by_date
-                        .get(&date)
-                        .and_then(|symbol_ids| {
-                            find_by_symbol_id_with_preferred_index(
-                                rows,
-                                symbol_ids,
-                                symbol_id,
-                                market_index,
-                            )
-                        })
-                })
-        });
-
-        SymbolSnapshotRefs {
-            market,
-            factor,
-            candidate,
-        }
-    }
-
     pub(crate) fn market_standard_rolling_means_by_symbol_id(
         &self,
         date: NaiveDate,
@@ -2033,10 +1964,13 @@ impl DataSet {
         volume_lookbacks: &[usize; 5],
         include_now: bool,
     ) -> StandardRollingMeans {
+        // Both series are built from the same market-date sequence. Reuse the
+        // indexed boundary lookup instead of repeating it for close and volume.
+        let series_end = self.market_series_end_index_by_symbol_id(date, symbol_id, include_now);
         let close = if close_lookbacks.iter().any(|lookback| *lookback > 0) {
             self.adjusted_close_series_by_symbol_id(symbol_id)
                 .map(|series| {
-                    self.market_series_end_index_by_symbol_id(date, symbol_id, include_now)
+                    series_end
                         .map(|end| series.moving_averages_at_end(end, close_lookbacks))
                         .unwrap_or_else(|| series.moving_averages(date, close_lookbacks, include_now))
                 })
@@ -2047,7 +1981,7 @@ impl DataSet {
         let volume = if volume_lookbacks.iter().any(|lookback| *lookback > 0) {
             self.market_series_by_symbol_id(symbol_id)
                 .map(|series| {
-                    self.market_series_end_index_by_symbol_id(date, symbol_id, include_now)
+                    series_end
                         .map(|end| series.volume_moving_averages_at_end(end, volume_lookbacks))
                         .unwrap_or_else(|| {
                             series.volume_moving_averages(date, volume_lookbacks, include_now)
@@ -4155,13 +4089,6 @@ fn find_by_symbol_id<'a, T>(rows: &'a [T], symbol_ids: &[u32], symbol_id: u32) -
     find_by_symbol_id_with_preferred_index(rows, symbol_ids, symbol_id, None)
 }
 
-fn symbol_id_index(rows_len: usize, symbol_ids: &[u32], symbol_id: u32) -> Option<usize> {
-    if rows_len != symbol_ids.len() {
-        return None;
-    }
-    symbol_ids.binary_search(&symbol_id).ok()
-}
-
 fn find_by_symbol_id_with_preferred_index<'a, T>(
     rows: &'a [T],
     symbol_ids: &[u32],
@@ -4786,7 +4713,7 @@ mod tests {
     }
 
     #[test]
-    fn combined_symbol_snapshot_lookup_uses_alignment_and_falls_back_for_sparse_rows() {
+    fn direct_symbol_id_snapshot_lookups_preserve_alignment_for_sparse_rows() {
         let date = NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap();
         let instrument = |symbol: &str| Instrument {
             symbol: symbol.to_string(),
@@ -4848,29 +4775,30 @@ mod tests {
 
         for symbol in ["000001.SZ", "600000.SH"] {
             let symbol_id = data.symbol_id(symbol).unwrap();
-            let combined = data.symbol_snapshots_by_id(date, symbol_id);
             assert_eq!(
-                combined.market.map(|row| row.symbol.as_str()),
                 data.market_by_symbol_id(date, symbol_id)
-                    .map(|row| row.symbol.as_str())
+                    .map(|row| row.symbol.as_str()),
+                Some(symbol)
             );
             assert_eq!(
-                combined.factor.map(|row| row.symbol.as_str()),
                 data.factor_by_symbol_id(date, symbol_id)
-                    .map(|row| row.symbol.as_str())
+                    .map(|row| row.symbol.as_str()),
+                Some(symbol)
             );
             assert_eq!(
-                combined.candidate.map(|row| row.symbol.as_str()),
                 data.candidate_by_symbol_id(date, symbol_id)
-                    .map(|row| row.symbol.as_str())
+                    .map(|row| row.symbol.as_str()),
+                Some(symbol)
             );
         }
 
         let signal_id = data.symbol_id("000300.SH").unwrap();
-        let signal = data.symbol_snapshots_by_id(date, signal_id);
+        assert_eq!(
-        assert_eq!(signal.market.map(|row| row.symbol.as_str()), Some("000300.SH"));
+            data.market_by_symbol_id(date, signal_id).map(|row| row.symbol.as_str()),
-        assert!(signal.factor.is_none());
+            Some("000300.SH")
-        assert!(signal.candidate.is_none());
+        );
+        assert!(data.factor_by_symbol_id(date, signal_id).is_none());
+        assert!(data.candidate_by_symbol_id(date, signal_id).is_none());
     }
 
     #[test]

crates/fidc-core/src/platform_expr_strategy.rs

@@ -990,6 +990,7 @@ pub struct PlatformExprStrategy {
     stock_rolling_requirements: StockRollingRequirements,
     stock_extra_factors_required: bool,
     stock_extra_factor_identifiers: BTreeSet<String>,
+    stock_extra_factor_map_required: bool,
     stock_text_factors_required: bool,
     stock_state_cache_date: RefCell<Option<NaiveDate>>,
     stock_state_cache: RefCell<
@@ -1267,6 +1268,7 @@ impl PlatformExprStrategy {
             Self::stock_extra_factors_required_for_config(&config, &prelude_declared_identifiers);
         let stock_extra_factor_identifiers =
             Self::stock_extra_factor_identifiers_for_config(&config, &prelude_declared_identifiers);
+        let stock_extra_factor_map_required = Self::stock_extra_factor_map_required_for_config(&config);
         let stock_text_factors_required = Self::stock_text_factors_required_for_config(
             &config,
             &normalized_stock_filter_expr,
@@ -1304,6 +1306,7 @@ impl PlatformExprStrategy {
             stock_rolling_requirements,
             stock_extra_factors_required,
             stock_extra_factor_identifiers,
+            stock_extra_factor_map_required,
             stock_text_factors_required,
             stock_state_cache_date: RefCell::new(None),
             stock_state_cache: RefCell::new(AHashMap::new()),
@@ -3900,28 +3903,28 @@ impl PlatformExprStrategy {
             return Ok(Arc::clone(state));
         }
 
-        let execution_snapshots = ctx.data.symbol_snapshots_by_id(date, symbol_id);
+        let market = ctx.data.market_by_symbol_id(date, symbol_id).ok_or_else(|| {
-        let market = execution_snapshots.market.ok_or_else(|| {
             BacktestError::Data(crate::data::DataSetError::MissingSnapshot {
                 kind: "market",
                 date,
                 symbol: symbol.to_string(),
             })
         })?;
-        let candidate = execution_snapshots.candidate.ok_or_else(|| {
+        let candidate = ctx.data.candidate_by_symbol_id(date, symbol_id).ok_or_else(|| {
             BacktestError::Data(crate::data::DataSetError::MissingSnapshot {
                 kind: "candidate",
                 date,
                 symbol: symbol.to_string(),
             })
         })?;
-        let factor_snapshots = if factor_date == date {
+        let feature_market = if factor_date == date {
-            execution_snapshots
+            market
         } else {
-            ctx.data.symbol_snapshots_by_id(factor_date, symbol_id)
+            ctx.data
+                .market_by_symbol_id(factor_date, symbol_id)
+                .unwrap_or(market)
         };
-        let feature_market = factor_snapshots.market.unwrap_or(market);
+        let factor = ctx.data.factor_by_symbol_id(factor_date, symbol_id).ok_or_else(|| {
-        let factor = factor_snapshots.factor.ok_or_else(|| {
             BacktestError::Data(crate::data::DataSetError::MissingSnapshot {
                 kind: "factor",
                 date: factor_date,
@@ -4055,6 +4058,10 @@ impl PlatformExprStrategy {
             factor
                 .extra_factors
                 .iter()
+                .filter(|(field, _)| {
+                    self.stock_extra_factor_map_required
+                        || self.stock_extra_factor_identifiers.contains(field.as_ref())
+                })
                 .map(|(field, value)| (field.to_string(), *value))
                 .collect()
         } else {
@@ -9219,6 +9226,56 @@ impl PlatformExprStrategy {
             .unwrap_or_else(|| self.field_value(candidate)))
     }
 
+    fn rank_reuses_market_cap_order(&self) -> bool {
+        self.config.rank_expr.trim().is_empty()
+            && !self.config.rank_desc
+            && matches!(
+                self.config.rank_by.trim(),
+                "market_cap" | "market_cap_bn"
+            )
+    }
+
+    fn selection_candidate_passes_filters(
+        &self,
+        ctx: &StrategyContext<'_>,
+        date: NaiveDate,
+        day: &DayExpressionState,
+        candidate: &EligibleUniverseSnapshot,
+        stock: &StockExpressionState,
+        diagnostics: &mut Vec<String>,
+    ) -> Result<bool, BacktestError> {
+        if !ctx.is_lagged_execution()
+            && let Some(reason) = self.stock_selection_limit_rejection_reason(stock)
+        {
+            if diagnostics.len() < 12 {
+                diagnostics.push(format!("{} rejected by {}", candidate.symbol, reason));
+            }
+            return Ok(false);
+        }
+        if !self.stock_passes_expr(ctx, day, stock)? {
+            if diagnostics.len() < 12 {
+                diagnostics.push(format!("{} rejected by stock_expr", candidate.symbol));
+            }
+            return Ok(false);
+        }
+        if self.config.stop_take_reference_price_mode
+            == PlatformStopTakeReferencePriceMode::SignalDayPostAdjustedClose
+            && ctx
+                .data
+                .market_latest_back_adjusted_close(date, &candidate.symbol)
+                .is_none()
+        {
+            if diagnostics.len() < 12 {
+                diagnostics.push(format!(
+                    "{} rejected by missing signal-day post-adjusted close",
+                    candidate.symbol
+                ));
+            }
+            return Ok(false);
+        }
+        Ok(true)
+    }
+
     fn can_sell_position(&self, ctx: &StrategyContext<'_>, date: NaiveDate, symbol: &str) -> bool {
         self.can_sell_position_at_time(ctx, date, symbol, None)
     }
@@ -9364,6 +9421,41 @@ impl PlatformExprStrategy {
             universe_factor_date,
             5,
         );
+
+        // The universe is already stably ordered by market cap.  When the
+        // strategy asks for that exact ascending order and does not need a
+        // complete ranking for replacement limiting, select directly from the
+        // ordered stream instead of materializing a second candidate vector.
+        if self.rank_reuses_market_cap_order() && self.config.daily_replacement_limit == 0 {
+            let mut selected = Vec::with_capacity(limit.min(universe.len()));
+            for candidate in universe {
+                let stock = self.selection_stock_state_with_factor_date(
+                    ctx,
+                    date,
+                    stock_factor_date,
+                    &candidate.symbol,
+                )?;
+                let field_value = self.selection_field_value(&candidate, &stock);
+                if !field_value.is_finite() || field_value < band_low || field_value > band_high {
+                    continue;
+                }
+                if self.selection_candidate_passes_filters(
+                    ctx,
+                    date,
+                    day,
+                    &candidate,
+                    &stock,
+                    &mut diagnostics,
+                )? {
+                    selected.push(candidate.symbol);
+                    if selected.len() >= limit {
+                        break;
+                    }
+                }
+            }
+            return Ok((selected, diagnostics, risk_decisions));
+        }
+
         let mut candidates = Vec::new();
         let mut missing_rank_count = 0usize;
         let mut missing_rank_examples = Vec::new();
@@ -9429,54 +9521,37 @@ impl PlatformExprStrategy {
             }
             candidates.push((candidate, stock, rank_value));
         }
-        candidates.sort_by(|lhs, rhs| {
+        if !self.rank_reuses_market_cap_order() {
-            let lhs_value = lhs.2;
+            candidates.sort_by(|lhs, rhs| {
-            let rhs_value = rhs.2;
+                let lhs_value = lhs.2;
-            let ordering = if self.config.rank_desc {
+                let rhs_value = rhs.2;
-                rhs_value
+                let ordering = if self.config.rank_desc {
-                    .partial_cmp(&lhs_value)
+                    rhs_value
-                    .unwrap_or(std::cmp::Ordering::Equal)
+                        .partial_cmp(&lhs_value)
-            } else {
+                        .unwrap_or(std::cmp::Ordering::Equal)
-                lhs_value
+                } else {
-                    .partial_cmp(&rhs_value)
+                    lhs_value
-                    .unwrap_or(std::cmp::Ordering::Equal)
+                        .partial_cmp(&rhs_value)
-            };
+                        .unwrap_or(std::cmp::Ordering::Equal)
-            if ordering == std::cmp::Ordering::Equal {
+                };
-                lhs.0.symbol.cmp(&rhs.0.symbol)
+                if ordering == std::cmp::Ordering::Equal {
-            } else {
+                    lhs.0.symbol.cmp(&rhs.0.symbol)
-                ordering
+                } else {
-            }
+                    ordering
-        });
+                }
+            });
+        }
 
         let mut selected = Vec::new();
         for (candidate, stock, _) in candidates {
-            if !ctx.is_lagged_execution()
+            if !self.selection_candidate_passes_filters(
-                && let Some(reason) = self.stock_selection_limit_rejection_reason(&stock)
+                ctx,
-            {
+                date,
-                if diagnostics.len() < 12 {
+                day,
-                    diagnostics.push(format!("{} rejected by {}", candidate.symbol, reason));
+                &candidate,
-                }
+                &stock,
-                continue;
+                &mut diagnostics,
-            }
+            )? {
-            if !self.stock_passes_expr(ctx, day, &stock)? {
-                if diagnostics.len() < 12 {
-                    diagnostics.push(format!("{} rejected by stock_expr", candidate.symbol));
-                }
-                continue;
-            }
-            if self.config.stop_take_reference_price_mode
-                == PlatformStopTakeReferencePriceMode::SignalDayPostAdjustedClose
-                && ctx
-                    .data
-                    .market_latest_back_adjusted_close(date, &candidate.symbol)
-                    .is_none()
-            {
-                if diagnostics.len() < 12 {
-                    diagnostics.push(format!(
-                        "{} rejected by missing signal-day post-adjusted close",
-                        candidate.symbol
-                    ));
-                }
                 continue;
             }
             selected.push(candidate.symbol.clone());
@@ -9597,6 +9672,30 @@ impl PlatformExprStrategy {
         .any(|expr| Self::expr_requires_stock_extra_factors(expr, prelude_declared_identifiers))
     }
 
+    fn stock_extra_factor_map_required_for_config(config: &PlatformExprStrategyConfig) -> bool {
+        // A dynamic factors/factor map is part of the expression contract: its
+        // caller may access any published field by key. Keep the full map for
+        // that case and for explicit actions whose predicates are evaluated at
+        // runtime; direct field references use the projected identifier set.
+        if Self::has_stock_explicit_actions(config) {
+            return true;
+        }
+        [
+            config.prelude.as_str(),
+            config.stock_filter_expr.as_str(),
+            config.buy_scale_expr.as_str(),
+            config.stop_loss_expr.as_str(),
+            config.take_profit_expr.as_str(),
+            config.rank_expr.as_str(),
+            config.market_cap_field.as_str(),
+            config.rank_by.as_str(),
+        ]
+        .into_iter()
+        .map(Self::normalize_expr)
+        .map(|expr| Self::extract_identifier_candidates(&expr))
+        .any(|identifiers| identifiers.contains("factors") || identifiers.contains("factor"))
+    }
+
     fn stock_extra_factor_identifiers_for_config(
         config: &PlatformExprStrategyConfig,
         prelude_declared_identifiers: &BTreeSet<String>,
@@ -13749,6 +13848,28 @@ mod tests {
         );
     }
 
+    #[test]
+    fn market_cap_ascending_rank_reuses_only_equivalent_universe_order() {
+        let mut cfg = PlatformExprStrategyConfig::microcap_rotation();
+        cfg.rank_by = "market_cap".to_string();
+        let strategy = PlatformExprStrategy::new(cfg.clone());
+        assert!(strategy.rank_reuses_market_cap_order());
+
+        cfg.rank_by = "market_cap_bn".to_string();
+        assert!(PlatformExprStrategy::new(cfg.clone()).rank_reuses_market_cap_order());
+
+        cfg.rank_desc = true;
+        assert!(!PlatformExprStrategy::new(cfg.clone()).rank_reuses_market_cap_order());
+
+        cfg.rank_desc = false;
+        cfg.rank_expr = "close".to_string();
+        assert!(!PlatformExprStrategy::new(cfg.clone()).rank_reuses_market_cap_order());
+
+        cfg.rank_expr.clear();
+        cfg.rank_by = "free_float_cap".to_string();
+        assert!(!PlatformExprStrategy::new(cfg).rank_reuses_market_cap_order());
+    }
+
     #[test]
     fn current_rolling_helpers_do_not_load_factor_maps_or_decision_rollings() {
         let mut cfg = PlatformExprStrategyConfig::microcap_rotation();
@@ -13831,7 +13952,10 @@ mod tests {
                     pe_ttm: 8.0,
                     turnover_ratio: Some(1.0),
                     effective_turnover_ratio: Some(1.0),
-                    extra_factors: BTreeMap::from([("model_score".into(), 2.0)]),
+                    extra_factors: BTreeMap::from([
+                        ("model_score".into(), 2.0),
+                        ("unused_factor".into(), 9.0),
+                    ]),
                 },
             ],
             symbols
@@ -13901,6 +14025,25 @@ mod tests {
                 .stock_passes_expr(&ctx, &day, &present_stock)
                 .expect("present stock filter")
         );
+        assert!(!strategy.stock_extra_factor_map_required);
+        assert!(present_stock.extra_factors.contains_key("model_score"));
+        assert!(!present_stock.extra_factors.contains_key("unused_factor"));
+
+        let mut map_cfg = PlatformExprStrategyConfig::microcap_rotation();
+        map_cfg.signal_symbol = present_symbol.to_string();
+        map_cfg.stock_filter_expr = "factors[\"unused_factor\"] > 0".to_string();
+        let map_strategy = PlatformExprStrategy::new(map_cfg);
+        assert!(map_strategy.stock_extra_factor_map_required);
+        let map_stock = map_strategy
+            .stock_state_with_factor_date(&ctx, date, date, present_symbol)
+            .expect("factor map stock state");
+        assert!(map_stock.extra_factors.contains_key("unused_factor"));
+        let map_day = map_strategy.day_state(&ctx, date).expect("factor map day state");
+        assert!(
+            map_strategy
+                .stock_passes_expr(&ctx, &map_day, &map_stock)
+                .expect("factor map filter")
+        );
     }
 
     #[test]