diff --git a/crates/fidc-core/src/strategy.rs b/crates/fidc-core/src/strategy.rs index bb619b6..461f50c 100644 --- a/crates/fidc-core/src/strategy.rs +++ b/crates/fidc-core/src/strategy.rs @@ -1,8 +1,4 @@ use std::collections::{BTreeMap, BTreeSet}; -use std::env; -use std::fs; -use std::path::{Path, PathBuf}; -use std::sync::OnceLock; use chrono::{Datelike, Duration, NaiveDate, NaiveDateTime, NaiveTime}; @@ -1694,12 +1690,6 @@ pub struct OmniMicroCapStrategy { config: OmniMicroCapConfig, } -#[derive(Debug, Clone)] -struct OmniTruthStockLists { - source_path: String, - symbols_by_date: BTreeMap>, -} - #[derive(Default)] struct ProjectedExecutionState { execution_cursors: BTreeMap, @@ -1719,23 +1709,6 @@ impl OmniMicroCapStrategy { Self { config } } - fn truth_stock_list_for_date(&self, date: NaiveDate) -> Option<&Vec> { - omni_truth_stock_lists() - .as_ref() - .and_then(|lists| lists.symbols_by_date.get(&date)) - } - - fn truth_stock_list_source_path(&self) -> Option<&str> { - omni_truth_stock_lists() - .as_ref() - .map(|lists| lists.source_path.as_str()) - } - - fn truth_selection_contains(&self, date: NaiveDate, symbol: &str) -> bool { - self.truth_stock_list_for_date(date) - .is_some_and(|symbols| symbols.iter().any(|item| item == symbol)) - } - fn stop_loss_tolerance(&self, market: &crate::data::DailyMarketSnapshot) -> f64 { let _ = market; 0.0 @@ -2438,9 +2411,7 @@ impl OmniMicroCapStrategy { ) { return Ok(Some(reason.to_string())); } - if !self.truth_selection_contains(date, symbol) - && !self.stock_passes_ma_filter(ctx, date, symbol) - { + if !self.stock_passes_ma_filter(ctx, date, symbol) { return Ok(Some("ma_filter".to_string())); } Ok(None) @@ -2511,78 +2482,6 @@ impl OmniMicroCapStrategy { band_low: f64, band_high: f64, ) -> Result<(Vec, Vec), BacktestError> { - if let Some(truth_symbols) = self.truth_stock_list_for_date(date) { - let mut diagnostics = vec![format!( - "selection_source=truth_csv path={} truth_candidates={}", - self.truth_stock_list_source_path().unwrap_or(""), - truth_symbols.len() - )]; - let mut selected = Vec::new(); - let mut selected_set = BTreeSet::new(); - let mut truth_selected = 0usize; - - for symbol in truth_symbols { - if selected.len() >= self.config.stocknum { - break; - } - if !selected_set.insert(symbol.clone()) { - continue; - } - if ctx.has_dynamic_universe() && !ctx.dynamic_universe_contains(symbol) { - selected_set.remove(symbol); - if diagnostics.len() < 14 { - diagnostics.push(format!("truth {} rejected by dynamic_universe", symbol)); - } - continue; - } - if let Some(reason) = self.buy_rejection_reason(ctx, date, symbol)? { - selected_set.remove(symbol); - if diagnostics.len() < 14 { - diagnostics.push(format!("truth {} rejected by {}", symbol, reason)); - } - continue; - } - selected.push(symbol.clone()); - truth_selected += 1; - } - - if selected.len() < self.config.stocknum { - let universe = - ctx.eligible_universe_on_with_risk_config(date, &self.config.risk_config); - let start = lower_bound_eligible(&universe, band_low); - for candidate in universe.iter().skip(start) { - if candidate.market_cap_bn > band_high { - break; - } - if selected.len() >= self.config.stocknum { - break; - } - if !selected_set.insert(candidate.symbol.clone()) { - continue; - } - if let Some(reason) = self.buy_rejection_reason(ctx, date, &candidate.symbol)? { - selected_set.remove(&candidate.symbol); - if diagnostics.len() < 18 { - diagnostics.push(format!( - "fallback {} rejected by {}", - candidate.symbol, reason - )); - } - continue; - } - selected.push(candidate.symbol.clone()); - } - } - - diagnostics.push(format!( - "truth_selected={} fallback_selected={} requested={}", - truth_selected, - selected.len().saturating_sub(truth_selected), - self.config.stocknum - )); - return Ok((selected, diagnostics)); - } - let universe = ctx.eligible_universe_on_with_risk_config(date, &self.config.risk_config); let mut diagnostics = Vec::new(); let mut selected = Vec::new(); @@ -2609,156 +2508,6 @@ impl OmniMicroCapStrategy { } } -fn omni_truth_stock_lists() -> &'static Option { - static LISTS: OnceLock> = OnceLock::new(); - LISTS.get_or_init(load_omni_truth_stock_lists) -} - -fn load_omni_truth_stock_lists() -> Option { - for path in omni_truth_stock_list_candidates() { - if !path.is_file() { - continue; - } - if let Ok(Some(lists)) = load_omni_truth_stock_lists_from_path(&path) { - return Some(lists); - } - } - None -} - -fn omni_truth_stock_list_candidates() -> Vec { - let mut candidates = Vec::new(); - for key in [ - "FIDC_BT_TRUTH_STOCK_LIST_CSV", - "OMNI_BT_TRUTH_STOCK_LIST_CSV", - "OMNI_BACKTEST_TRUTH_STOCK_LIST_CSV", - ] { - if let Ok(value) = env::var(key) { - let trimmed = value.trim(); - if !trimmed.is_empty() { - push_unique_truth_path(&mut candidates, PathBuf::from(trimmed)); - } - } - } - candidates -} - -fn push_unique_truth_path(paths: &mut Vec, candidate: PathBuf) { - if !paths.iter().any(|existing| existing == &candidate) { - paths.push(candidate); - } -} - -fn load_omni_truth_stock_lists_from_path( - path: &Path, -) -> Result, String> { - let text = fs::read_to_string(path) - .map_err(|error| format!("read {} failed: {}", path.display(), error))?; - let mut lines = text.lines().filter(|line| !line.trim().is_empty()); - let Some(header_line) = lines.next() else { - return Ok(None); - }; - let headers = split_simple_csv_line(header_line.trim_start_matches('\u{feff}')); - let trade_date_idx = headers - .iter() - .position(|field| field == "trade_date") - .ok_or_else(|| format!("missing trade_date column in {}", path.display()))?; - let symbol_idx = headers - .iter() - .position(|field| field == "symbol") - .ok_or_else(|| format!("missing symbol column in {}", path.display()))?; - let rank_idx = headers - .iter() - .position(|field| field == "rank") - .or_else(|| headers.iter().position(|field| field == "index")); - - let mut rows_by_date: BTreeMap> = BTreeMap::new(); - for (offset, line) in lines.enumerate() { - let cols = split_simple_csv_line(line); - let date_raw = cols - .get(trade_date_idx) - .ok_or_else(|| format!("missing trade_date at {}:{}", path.display(), offset + 2))?; - let symbol_raw = cols - .get(symbol_idx) - .ok_or_else(|| format!("missing symbol at {}:{}", path.display(), offset + 2))?; - let trade_date = NaiveDate::parse_from_str(date_raw, "%Y-%m-%d").map_err(|error| { - format!( - "invalid trade_date at {}:{}: {}", - path.display(), - offset + 2, - error - ) - })?; - let Some(symbol) = normalize_truth_symbol(symbol_raw) else { - return Err(format!( - "invalid symbol at {}:{}", - path.display(), - offset + 2 - )); - }; - let rank = rank_idx - .and_then(|idx| cols.get(idx)) - .and_then(|value| value.parse::().ok()) - .unwrap_or_else(|| { - rows_by_date - .get(&trade_date) - .map(|items| items.len() + 1) - .unwrap_or(1) - }); - rows_by_date - .entry(trade_date) - .or_default() - .push((rank.max(1), symbol)); - } - - if rows_by_date.is_empty() { - return Ok(None); - } - - let symbols_by_date = rows_by_date - .into_iter() - .map(|(date, mut rows)| { - rows.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1))); - let mut seen = BTreeSet::new(); - let ordered = rows - .into_iter() - .filter_map(|(_, symbol)| { - if seen.insert(symbol.clone()) { - Some(symbol) - } else { - None - } - }) - .collect::>(); - (date, ordered) - }) - .collect::>(); - - Ok(Some(OmniTruthStockLists { - source_path: path.display().to_string(), - symbols_by_date, - })) -} - -fn split_simple_csv_line(line: &str) -> Vec { - line.split(',') - .map(|field| field.trim().trim_matches('"').to_string()) - .collect() -} - -fn normalize_truth_symbol(raw: &str) -> Option { - let normalized = raw - .trim() - .to_ascii_uppercase() - .replace(".XSHG", ".SH") - .replace(".XSHE", ".SZ"); - if normalized.is_empty() { - None - } else { - Some(normalized) - } -} - impl Strategy for OmniMicroCapStrategy { fn name(&self) -> &str { self.config.strategy_name.as_str() @@ -3021,51 +2770,6 @@ fn lower_bound_eligible(rows: &[crate::data::EligibleUniverseSnapshot], target: mod tests { use super::*; use crate::{BenchmarkSnapshot, CandidateEligibility, DailyFactorSnapshot}; - use std::time::{SystemTime, UNIX_EPOCH}; - - fn temp_csv_path(name: &str) -> PathBuf { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - env::temp_dir().join(format!("{}_{}_{}.csv", name, std::process::id(), nanos)) - } - - #[test] - fn load_truth_stock_lists_preserves_rank_order() { - let path = temp_csv_path("omni_truth_list"); - fs::write( - &path, - "trade_date,index,symbol\n2025-01-02,2,300935.SZ\n2025-01-02,1,300321.XSHE\n2025-01-02,1,300321.SZ\n", - ) - .unwrap(); - - let lists = load_omni_truth_stock_lists_from_path(&path) - .unwrap() - .unwrap(); - fs::remove_file(&path).ok(); - - let symbols = lists - .symbols_by_date - .get(&NaiveDate::from_ymd_opt(2025, 1, 2).unwrap()) - .unwrap(); - assert_eq!( - symbols, - &vec!["300321.SZ".to_string(), "300935.SZ".to_string()] - ); - } - - #[test] - fn normalize_truth_symbol_maps_external_platform_suffixes() { - assert_eq!( - normalize_truth_symbol("300321.XSHE").as_deref(), - Some("300321.SZ") - ); - assert_eq!( - normalize_truth_symbol("603657.XSHG").as_deref(), - Some("603657.SH") - ); - } #[test] fn omni_microcap_projection_uses_configured_trading_cost() { diff --git a/scripts/verify-no-legacy-data-source.sh b/scripts/verify-no-legacy-data-source.sh index 7242f91..f1ba2ae 100755 --- a/scripts/verify-no-legacy-data-source.sh +++ b/scripts/verify-no-legacy-data-source.sh @@ -79,4 +79,23 @@ if [[ -n "$fused_hits" ]]; then fail "exported fused tables are not allowed in fidc-backtest-engine runtime; use Strategy Factory Source Lake source rows directly" "$fused_hits" fi +truth_csv_hits="$( + rg -n \ + --glob '!**/.git/**' \ + --glob '!**/target/**' \ + --glob '!**/docs/**' \ + --glob '!**/*.md' \ + --glob '!**/tests/**' \ + --glob '!**/*test*.rs' \ + --glob '!crates/fidc-core/src/strategy_ai.rs' \ + --glob '!scripts/verify-no-legacy-data-source.sh' \ + 'FIDC_BT_TRUTH_STOCK_LIST_CSV|OMNI_BT_TRUTH_STOCK_LIST_CSV|OMNI_BACKTEST_TRUTH_STOCK_LIST_CSV|selection_source=truth_csv|truth_stock_list|truth_csv' \ + crates Cargo.toml \ + 2>/dev/null || true +)" + +if [[ -n "$truth_csv_hits" ]]; then + fail "CSV truth stock-list overrides are not allowed in fidc-backtest-engine runtime; use Source Lake runtime spec selection only" "$truth_csv_hits" +fi + printf '[OK] fidc-backtest-engine has no legacy runtime data-source references\n'