From 0ed6752a73b15c46a6301b4b17d34de5e8cb1840 Mon Sep 17 00:00:00 2001
From: boris <shuaibo.zhang@hotmail.com>
Date: Sun, 13 Sep 2026 16:47:15 +0800
Subject: [PATCH] perf(engine): deduplicate daily factor names before
 allocating sorted output

---
 .../fidc-core/src/platform_expr_strategy.rs   | 47 ++++++++++++++-----
 1 file changed, 35 insertions(+), 12 deletions(-)

diff --git a/crates/fidc-core/src/platform_expr_strategy.rs b/crates/fidc-core/src/platform_expr_strategy.rs
index 66ac3c9..58eac3c 100644
--- a/crates/fidc-core/src/platform_expr_strategy.rs
+++ b/crates/fidc-core/src/platform_expr_strategy.rs
@@ -961,6 +961,16 @@ struct DayExpressionState {
     available_text_factor_names: BTreeSet<String>,
 }
 
+fn collect_available_factor_names<'a>(names: impl Iterator<Item = &'a str>) -> BTreeSet<String> {
+    // BTreeSet::from_iter first sorts a Vec containing every repeated name.
+    // The daily universe has many rows but usually few distinct factor fields.
+    let mut unique = BTreeSet::new();
+    for name in names {
+        unique.insert(name);
+    }
+    unique.into_iter().map(str::to_owned).collect()
+}
+
 #[derive(Debug, Clone)]
 struct StockExpressionState {
     symbol: Arc<str>,
@@ -4388,7 +4398,7 @@ impl PlatformExprStrategy {
             is_month_start: date.day() == 1,
             is_month_end,
             available_factor_names: if self.stock_extra_factors_required {
-                ctx.data
+                collect_available_factor_names(ctx.data
                     .factor_snapshot_rows_on(date)
                     .iter()
                     .flat_map(|row| {
@@ -4396,23 +4406,15 @@ impl PlatformExprStrategy {
                             row.adjustment_factor_backward1
                                 .map(|_| BACKWARD_ADJUSTMENT_FACTOR_FIELD),
                         )
-                    })
-                    .collect::<BTreeSet<_>>()
-                    .into_iter()
-                    .map(str::to_owned)
-                    .collect()
+                    }))
             } else {
                 BTreeSet::new()
             },
             available_text_factor_names: if self.stock_text_factors_required {
-                ctx.data
+                collect_available_factor_names(ctx.data
                     .factor_text_rows_on(date)
                     .iter()
-                    .map(|row| row.field.as_str())
-                    .collect::<BTreeSet<_>>()
-                    .into_iter()
-                    .map(str::to_owned)
-                    .collect()
+                    .map(|row| row.field.as_str()))
             } else {
                 BTreeSet::new()
             },
@@ -14594,6 +14596,27 @@ mod tests {
         NaiveDate::from_ymd_opt(year, month, day).expect("valid date")
     }
 
+    #[test]
+    fn available_factor_name_collection_preserves_sparse_and_repeated_fields() {
+        let fields = ["amount", "model_score", "amount", "adjustment_factor_backward1"];
+        let names = (0..5_000).flat_map(|_| fields.iter().copied());
+        let expected = names.clone().collect::<BTreeSet<_>>()
+            .into_iter().map(str::to_owned).collect::<BTreeSet<_>>();
+        assert_eq!(super::collect_available_factor_names(names), expected);
+        assert!(super::collect_available_factor_names(std::iter::empty()).is_empty());
+        assert_eq!(super::collect_available_factor_names(["today_only"].into_iter()),
+            BTreeSet::from(["today_only".to_string()]));
+    }
+
+    #[test]
+    fn available_factor_name_collection_preserves_wide_dynamic_field_identity() {
+        let fields = (0..4_000).map(|index| format!("dynamic_{index:04}"))
+            .chain(["Model_score".to_string(), "model_score".to_string()]).collect::<Vec<_>>();
+        let expected = fields.iter().cloned().collect::<BTreeSet<_>>();
+        let names = fields.iter().rev().chain(fields.iter()).map(String::as_str);
+        assert_eq!(super::collect_available_factor_names(names), expected);
+    }
+
     #[test]
     fn buy_filter_attaches_denials_without_rewriting_selection() {
         let prev = d(2025, 1, 2);