diff --git a/Cargo.lock b/Cargo.lock index 3e77046..6e47fbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,16 +37,6 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" -[[package]] -name = "bt-demo" -version = "0.1.0" -dependencies = [ - "chrono", - "fidc-core", - "serde", - "serde_json", -] - [[package]] name = "bumpalo" version = "3.20.2" diff --git a/Cargo.toml b/Cargo.toml index 548e42d..16857ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,6 @@ [workspace] members = [ "crates/fidc-core", - "crates/bt-demo", ] resolver = "2" diff --git a/README.md b/README.md index 4c59305..ff931ef 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,6 @@ . ├── Cargo.toml ├── crates -│ ├── bt-demo │ └── fidc-core │ └── src │ ├── broker.rs @@ -37,7 +36,6 @@ │ ├── scheduler.rs │ ├── strategy.rs │ └── strategy_ai.rs -├── data/demo └── docs ``` @@ -96,45 +94,9 @@ ## 运行方式 -默认运行仓库 demo 数据: +`fidc-backtest-engine` 不再维护本地 CSV demo、partitioned snapshot 目录或导出融合表作为运行入口。生产和集成回测由 `fidc-backtest-service` runner 创建 `DataSet`,数据来自 Strategy Factory Source Lake 的 Arrow/Parquet、manifest/data_epoch 缓存和运行时逻辑视图。 -```bash -cargo run --bin bt-demo -``` - -运行平台内置微盘策略: - -```bash -FIDC_BT_STRATEGY=omni-microcap \ -FIDC_BT_SIGNAL_SYMBOL=000001.SH \ -cargo run --release --bin bt-demo -``` - -接入真实分区 snapshot 目录: - -```bash -FIDC_BT_DATA_LAYOUT=partitioned \ -FIDC_BT_DATA_DIR=/path/to/snapshots \ -FIDC_BT_SIGNAL_SYMBOL=000001.SH \ -cargo run --bin bt-demo -``` - -约定目录结构: - -```text -snapshots/ -├── instruments.csv -├── benchmark/YYYY/MM/*.csv -├── market/YYYY/MM/*.csv -├── factors/YYYY/MM/*.csv -└── candidates/YYYY/MM/*.csv -``` - -运行后默认生成: - -- `output/demo/equity_curve.csv` -- `output/demo/trades.csv` -- `output/demo/holdings_summary.csv` +本仓库只保留核心库构建和测试入口: ## 测试与构建 diff --git a/crates/bt-demo/Cargo.toml b/crates/bt-demo/Cargo.toml deleted file mode 100644 index ac837e2..0000000 --- a/crates/bt-demo/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "bt-demo" -version.workspace = true -edition.workspace = true -license.workspace = true -authors.workspace = true - -[dependencies] -chrono = { workspace = true } -fidc-core = { path = "../fidc-core" } -serde = { workspace = true } -serde_json = "1" diff --git a/crates/bt-demo/src/main.rs b/crates/bt-demo/src/main.rs deleted file mode 100644 index 15238b6..0000000 --- a/crates/bt-demo/src/main.rs +++ /dev/null @@ -1,567 +0,0 @@ -use std::collections::BTreeSet; -use std::error::Error; -use std::fs; -use std::io::Write; -use std::path::{Path, PathBuf}; - -use chrono::{NaiveDate, NaiveTime}; -use fidc_core::{ - BacktestConfig, BacktestEngine, BenchmarkSnapshot, BrokerSimulator, ChinaAShareCostModel, - ChinaEquityRuleHooks, CnSmallCapRotationConfig, CnSmallCapRotationStrategy, DailyEquityPoint, - DataSet, FillEvent, HoldingSummary, OmniMicroCapConfig, OmniMicroCapStrategy, PortfolioState, - PriceField, Strategy, StrategyContext, -}; -use serde_json::json; - -fn main() -> Result<(), Box> { - let root = workspace_root(); - let data_dir = std::env::var("FIDC_BT_DATA_DIR") - .map(PathBuf::from) - .unwrap_or_else(|_| root.join("data/demo")); - let data_layout = std::env::var("FIDC_BT_DATA_LAYOUT").unwrap_or_else(|_| "flat".to_string()); - let output_dir = std::env::var("FIDC_BT_OUTPUT_DIR") - .map(PathBuf::from) - .unwrap_or_else(|_| root.join("output/demo")); - let json_output = std::env::var("FIDC_BT_JSON") - .map(|value| value == "1" || value.eq_ignore_ascii_case("true")) - .unwrap_or(false); - - fs::create_dir_all(&output_dir)?; - - let data = if data_layout == "partitioned" { - DataSet::from_partitioned_dir(&data_dir)? - } else { - DataSet::from_csv_dir(&data_dir)? - }; - let strategy_name = - std::env::var("FIDC_BT_STRATEGY").unwrap_or_else(|_| "cn-smallcap-rotation".to_string()); - let debug_date = std::env::var("FIDC_BT_DEBUG_DATE") - .ok() - .filter(|value| !value.trim().is_empty()) - .map(|value| NaiveDate::parse_from_str(value.trim(), "%Y-%m-%d")) - .transpose()?; - let decision_lag = std::env::var("FIDC_BT_DECISION_LAG") - .ok() - .and_then(|value| value.parse::().ok()); - let execution_price = - std::env::var("FIDC_BT_EXECUTION_PRICE") - .ok() - .map(|value| match value.as_str() { - "close" => PriceField::Close, - "last" => PriceField::Last, - _ => PriceField::Open, - }); - let initial_cash = std::env::var("FIDC_BT_INITIAL_CASH") - .ok() - .and_then(|value| value.parse::().ok()); - let start_date = std::env::var("FIDC_BT_START_DATE") - .ok() - .filter(|value| !value.trim().is_empty()) - .map(|value| NaiveDate::parse_from_str(value.trim(), "%Y-%m-%d")) - .transpose()?; - let end_date = std::env::var("FIDC_BT_END_DATE") - .ok() - .filter(|value| !value.trim().is_empty()) - .map(|value| NaiveDate::parse_from_str(value.trim(), "%Y-%m-%d")) - .transpose()?; - let mut config = BacktestConfig { - initial_cash: initial_cash.unwrap_or(1_000_000.0), - benchmark_code: data.benchmark_code().to_string(), - start_date, - end_date, - decision_lag_trading_days: 1, - execution_price_field: PriceField::Open, - }; - let result = match strategy_name.as_str() { - "cn-smallcap-rotation" | "cn-dyn-smallcap-band" => { - let mut strategy_cfg = if strategy_name == "cn-dyn-smallcap-band" { - CnSmallCapRotationConfig::cn_dyn_smallcap_band() - } else { - CnSmallCapRotationConfig::demo() - }; - if strategy_cfg.strategy_name == "cn-smallcap-rotation" { - strategy_cfg.base_index_level = 3000.0; - strategy_cfg.base_cap_floor = 38.0; - strategy_cfg.cap_span = 25.0; - } - if let Ok(signal_symbol) = std::env::var("FIDC_BT_SIGNAL_SYMBOL") { - if !signal_symbol.trim().is_empty() { - strategy_cfg.signal_symbol = Some(signal_symbol); - } - } - config.decision_lag_trading_days = decision_lag.unwrap_or(1); - config.execution_price_field = execution_price.unwrap_or(PriceField::Open); - let strategy = CnSmallCapRotationStrategy::new(strategy_cfg); - let broker = BrokerSimulator::new_with_execution_price( - ChinaAShareCostModel::default(), - ChinaEquityRuleHooks::default(), - config.execution_price_field, - ); - let mut engine = BacktestEngine::new(data, strategy, broker, config); - engine.run()? - } - "aiquant-v104" => { - let mut strategy_cfg = OmniMicroCapConfig::aiquant_v104(); - if let Ok(signal_symbol) = std::env::var("FIDC_BT_SIGNAL_SYMBOL") { - if !signal_symbol.trim().is_empty() { - strategy_cfg.benchmark_signal_symbol = signal_symbol; - } - } - if let Some(date) = debug_date { - let eligible = data.eligible_universe_on(date); - eprintln!( - "DEBUG eligible_universe_on {} count={}", - date, - eligible.len() - ); - for row in eligible.iter().take(20) { - eprintln!(" {} {:.6}", row.symbol, row.market_cap_bn); - } - let mut debug_strategy = OmniMicroCapStrategy::new(strategy_cfg.clone()); - let debug_subscriptions = BTreeSet::new(); - let decision = debug_strategy.on_day(&StrategyContext { - execution_date: date, - decision_date: date, - decision_index: 1, - data: &data, - portfolio: &PortfolioState::new(20_000.0), - futures_account: None, - open_orders: &[], - dynamic_universe: None, - subscriptions: &debug_subscriptions, - process_events: &[], - active_process_event: None, - active_datetime: None, - order_events: &[], - fills: &[], - })?; - eprintln!("DEBUG notes={:?}", decision.notes); - eprintln!("DEBUG diagnostics={:?}", decision.diagnostics); - return Ok(()); - } - config.decision_lag_trading_days = decision_lag.unwrap_or(1); - config.execution_price_field = execution_price.unwrap_or(PriceField::Close); - config.initial_cash = initial_cash.unwrap_or(20_000.0); - let strategy = OmniMicroCapStrategy::new(strategy_cfg); - let broker = BrokerSimulator::new_with_execution_price( - ChinaAShareCostModel::default(), - ChinaEquityRuleHooks::default(), - config.execution_price_field, - ); - let mut engine = BacktestEngine::new(data, strategy, broker, config); - engine.run()? - } - _ => { - let mut strategy_cfg = OmniMicroCapConfig::omni_microcap(); - if let Ok(signal_symbol) = std::env::var("FIDC_BT_SIGNAL_SYMBOL") { - if !signal_symbol.trim().is_empty() { - strategy_cfg.benchmark_signal_symbol = signal_symbol; - } - } - if let Some(date) = debug_date { - let eligible = data.eligible_universe_on(date); - eprintln!( - "DEBUG eligible_universe_on {} count={}", - date, - eligible.len() - ); - for row in eligible.iter().take(20) { - eprintln!(" {} {:.6}", row.symbol, row.market_cap_bn); - } - let mut debug_strategy = OmniMicroCapStrategy::new(strategy_cfg.clone()); - let debug_subscriptions = BTreeSet::new(); - let decision = debug_strategy.on_day(&StrategyContext { - execution_date: date, - decision_date: date, - decision_index: 1, - data: &data, - portfolio: &PortfolioState::new(10_000_000.0), - futures_account: None, - open_orders: &[], - dynamic_universe: None, - subscriptions: &debug_subscriptions, - process_events: &[], - active_process_event: None, - active_datetime: None, - order_events: &[], - fills: &[], - })?; - eprintln!("DEBUG notes={:?}", decision.notes); - eprintln!("DEBUG diagnostics={:?}", decision.diagnostics); - return Ok(()); - } - config.decision_lag_trading_days = decision_lag.unwrap_or(0); - config.execution_price_field = execution_price.unwrap_or(PriceField::Last); - config.initial_cash = initial_cash.unwrap_or(10_000_000.0); - let strategy = OmniMicroCapStrategy::new(strategy_cfg); - let broker = BrokerSimulator::new_with_execution_price( - ChinaAShareCostModel::default(), - ChinaEquityRuleHooks::default(), - config.execution_price_field, - ) - .with_intraday_execution_start_time( - NaiveTime::parse_from_str("10:18:00", "%H:%M:%S").expect("valid 10:18:00"), - ) - .with_volume_limit(false) - .with_inactive_limit(false) - .with_liquidity_limit(false); - let mut engine = BacktestEngine::new(data, strategy, broker, config); - engine.run()? - } - }; - - write_equity_curve_csv(&output_dir.join("equity_curve.csv"), &result.equity_curve)?; - write_trades_csv(&output_dir.join("trades.csv"), &result.fills)?; - write_holdings_csv( - &output_dir.join("holdings_summary.csv"), - &result.holdings_summary, - )?; - - let summary = build_summary( - &result.strategy_name, - &result.equity_curve, - &result.fills, - &result.holdings_summary, - result.benchmark_series.last(), - &output_dir, - ); - - print_summary(&summary, &result.equity_curve, &result.holdings_summary); - println!("Artifacts written under {}", output_dir.display()); - if json_output { - println!("{}", serde_json::to_string(&summary)?); - } - Ok(()) -} - -fn workspace_root() -> PathBuf { - Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../..") - .canonicalize() - .expect("workspace root") -} - -fn write_equity_curve_csv(path: &Path, rows: &[DailyEquityPoint]) -> Result<(), Box> { - let mut file = fs::File::create(path)?; - writeln!( - file, - "date,cash,market_value,total_equity,benchmark_close,benchmark_prev_close,notes,diagnostics" - )?; - for row in rows { - writeln!( - file, - "{},{:.2},{:.2},{:.2},{:.2},{:.2},{},{}", - row.date, - row.cash, - row.market_value, - row.total_equity, - row.benchmark_close, - row.benchmark_prev_close, - sanitize_csv_field(&row.notes), - sanitize_csv_field(&row.diagnostics), - )?; - } - Ok(()) -} - -fn write_trades_csv(path: &Path, rows: &[FillEvent]) -> Result<(), Box> { - let mut file = fs::File::create(path)?; - writeln!( - file, - "date,symbol,side,quantity,price,gross_amount,commission,stamp_tax,net_cash_flow,reason" - )?; - for row in rows { - writeln!( - file, - "{},{},{:?},{},{:.2},{:.2},{:.2},{:.2},{:.2},{}", - row.date, - row.symbol, - row.side, - row.quantity, - row.price, - row.gross_amount, - row.commission, - row.stamp_tax, - row.net_cash_flow, - sanitize_csv_field(&row.reason), - )?; - } - Ok(()) -} - -fn write_holdings_csv(path: &Path, rows: &[HoldingSummary]) -> Result<(), Box> { - let mut file = fs::File::create(path)?; - writeln!( - file, - "date,symbol,quantity,average_cost,last_price,market_value,unrealized_pnl,realized_pnl" - )?; - for row in rows { - writeln!( - file, - "{},{},{},{:.2},{:.2},{:.2},{:.2},{:.2}", - row.date, - row.symbol, - row.quantity, - row.average_cost, - row.last_price, - row.market_value, - row.unrealized_pnl, - row.realized_pnl, - )?; - } - Ok(()) -} - -fn sanitize_csv_field(text: &str) -> String { - text.replace(',', ";") -} - -#[derive(Debug, serde::Serialize)] -struct RunSummary { - strategy: String, - start_date: String, - end_date: String, - start_equity: f64, - final_equity: f64, - total_return: f64, - trade_count: usize, - holding_count: usize, - benchmark_code: Option, - benchmark_last_close: Option, - output_dir: String, - diagnostics: serde_json::Value, - warnings: Vec, - equity_preview: Vec, - trades_preview: Vec, -} - -fn build_summary( - strategy_name: &str, - equity_curve: &[DailyEquityPoint], - fills: &[FillEvent], - holdings: &[HoldingSummary], - benchmark_last: Option<&BenchmarkSnapshot>, - output_dir: &Path, -) -> RunSummary { - let first = equity_curve.first(); - let last = equity_curve.last(); - let start_equity = first.map(|row| row.total_equity).unwrap_or_default(); - let final_equity = last.map(|row| row.total_equity).unwrap_or_default(); - let total_return = if start_equity.abs() < f64::EPSILON { - 0.0 - } else { - (final_equity / start_equity) - 1.0 - }; - - let diagnostics = extract_diagnostics(equity_curve); - let warnings = build_warnings(fills, holdings, &diagnostics); - let equity_preview = equity_curve - .iter() - .rev() - .take(5) - .collect::>() - .into_iter() - .rev() - .map(|row| { - json!({ - "date": row.date.to_string(), - "cash": row.cash, - "marketValue": row.market_value, - "totalEquity": row.total_equity, - "benchmarkClose": row.benchmark_close, - "benchmarkPrevClose": row.benchmark_prev_close, - "notes": row.notes, - "diagnostics": row.diagnostics, - }) - }) - .collect::>(); - let trades_preview = fills - .iter() - .rev() - .take(10) - .collect::>() - .into_iter() - .rev() - .map(|row| { - json!({ - "date": row.date.to_string(), - "symbol": row.symbol, - "side": format!("{:?}", row.side), - "quantity": row.quantity, - "price": row.price, - "grossAmount": row.gross_amount, - "netCashFlow": row.net_cash_flow, - "reason": row.reason, - }) - }) - .collect::>(); - - RunSummary { - strategy: strategy_name.to_string(), - start_date: first.map(|row| row.date.to_string()).unwrap_or_default(), - end_date: last.map(|row| row.date.to_string()).unwrap_or_default(), - start_equity, - final_equity, - total_return, - trade_count: fills.len(), - holding_count: holdings.len(), - benchmark_code: benchmark_last.map(|row| row.benchmark.clone()), - benchmark_last_close: benchmark_last.map(|row| row.close), - output_dir: output_dir.display().to_string(), - diagnostics, - warnings, - equity_preview, - trades_preview, - } -} - -fn extract_diagnostics(equity_curve: &[DailyEquityPoint]) -> serde_json::Value { - let last = equity_curve.last(); - let text = last.map(|row| row.diagnostics.as_str()).unwrap_or(""); - let notes = last.map(|row| row.notes.as_str()).unwrap_or(""); - let mut map = serde_json::Map::new(); - map.insert("latestText".to_string(), json!(text)); - map.insert("latestNotes".to_string(), json!(notes)); - map.insert("equityPointCount".to_string(), json!(equity_curve.len())); - - for part in text.split(" | ") { - let part = part.trim(); - if let Some(rest) = part.strip_prefix("selection_diag ") { - for token in rest.split_whitespace() { - if let Some((k, v)) = token.split_once('=') { - map.insert(k.to_string(), parse_diag_value(v)); - } - } - } else if let Some(rest) = part.strip_prefix("selection_band ") { - for token in rest.split_whitespace() { - if let Some((k, v)) = token.split_once('=') { - map.insert(k.to_string(), parse_diag_value(v)); - } - } - } else if let Some(rest) = - part.strip_prefix("market_cap_missing likely blocks selection; sample=") - { - map.insert( - "marketCapMissingSample".to_string(), - json!( - rest.split('|') - .filter(|s| !s.is_empty()) - .collect::>() - ), - ); - } else if let Some(rest) = part.strip_prefix("selection_rejections sample=") { - map.insert( - "selectionRejectionsSample".to_string(), - json!( - rest.split(" | ") - .filter(|s| !s.is_empty()) - .collect::>() - ), - ); - } else if let Some(rest) = part.strip_prefix("ma_filter_rejections sample=") { - map.insert( - "maFilterRejectionsSample".to_string(), - json!( - rest.split('|') - .filter(|s| !s.is_empty()) - .collect::>() - ), - ); - } else if let Some(rest) = part.strip_prefix("selected=") { - map.insert("selectedLine".to_string(), json!(rest)); - } - } - - serde_json::Value::Object(map) -} - -fn parse_diag_value(value: &str) -> serde_json::Value { - if let Ok(v) = value.parse::() { - return json!(v); - } - if let Ok(v) = value.parse::() { - return json!(v); - } - json!(value) -} - -fn build_warnings( - fills: &[FillEvent], - holdings: &[HoldingSummary], - diagnostics: &serde_json::Value, -) -> Vec { - let mut warnings = Vec::new(); - if fills.is_empty() { - warnings.push("本次回测没有产生任何成交。".to_string()); - } - if holdings.is_empty() { - warnings.push("期末没有持仓。".to_string()); - } - let selected_after_ma_is_empty = diagnostics - .get("selected_after_ma") - .and_then(|v| v.as_i64()) - .unwrap_or(0) - == 0; - if selected_after_ma_is_empty && fills.is_empty() && holdings.is_empty() { - warnings - .push("最终没有股票通过完整选股链路,结果为空时请优先查看 diagnostics。".to_string()); - } - if diagnostics - .get("market_cap_missing_count") - .and_then(|v| v.as_i64()) - .unwrap_or(0) - > 0 - { - warnings.push("存在 market_cap 缺失或非正值,当前会直接阻断该股票进入候选池。".to_string()); - } - warnings -} - -fn print_summary( - summary: &RunSummary, - equity_curve: &[DailyEquityPoint], - holdings: &[HoldingSummary], -) { - if equity_curve.is_empty() { - println!("No equity curve points generated."); - return; - } - - println!("Strategy: {}", summary.strategy); - println!("Start equity: {:.2}", summary.start_equity); - println!("Final equity: {:.2}", summary.final_equity); - println!("Total return: {:.2}%", summary.total_return * 100.0); - println!("Trades: {}", summary.trade_count); - println!("Final holdings: {}", summary.holding_count); - - if let (Some(code), Some(close)) = (&summary.benchmark_code, summary.benchmark_last_close) { - println!("Benchmark last close: {} {:.2}", code, close); - } - - println!("Recent equity points:"); - for point in equity_curve - .iter() - .rev() - .take(3) - .collect::>() - .into_iter() - .rev() - { - println!( - " {} equity {:.2} cash {:.2} mv {:.2}", - point.date, point.total_equity, point.cash, point.market_value - ); - } - - if holdings.is_empty() { - println!("No holdings at the end of the demo run."); - } else { - println!("Ending holdings:"); - for holding in holdings { - println!( - " {} qty {} mv {:.2} pnl {:.2}", - holding.symbol, holding.quantity, holding.market_value, holding.unrealized_pnl - ); - } - } -} diff --git a/crates/fidc-core/src/data.rs b/crates/fidc-core/src/data.rs index 1ba9d87..9041e7c 100644 --- a/crates/fidc-core/src/data.rs +++ b/crates/fidc-core/src/data.rs @@ -1,6 +1,4 @@ use std::collections::{BTreeMap, HashMap, HashSet}; -use std::fs; -use std::path::Path; use std::sync::{Arc, OnceLock, RwLock}; use chrono::{NaiveDate, NaiveDateTime}; @@ -8,7 +6,7 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::calendar::TradingCalendar; -use crate::futures::{FuturesCommissionType, FuturesTradingParameter}; +use crate::futures::FuturesTradingParameter; use crate::instrument::Instrument; use crate::risk_control::{ChinaAShareRiskControl, FidcRiskControlConfig}; @@ -58,18 +56,6 @@ mod datetime_format { #[derive(Debug, Error)] pub enum DataSetError { - #[error("failed to read file {path}: {source}")] - Io { - path: String, - #[source] - source: std::io::Error, - }, - #[error("invalid csv row in {path} at line {line}: {message}")] - InvalidRow { - path: String, - line: usize, - message: String, - }, #[error("benchmark file contains multiple benchmark codes")] MultipleBenchmarks, #[error("missing data for {kind} on {date} / {symbol}")] @@ -1006,96 +992,6 @@ pub struct DataSet { } impl DataSet { - pub fn from_csv_dir(path: &Path) -> Result { - let instruments = read_instruments(&path.join("instruments.csv"))?; - let market = read_market(&path.join("market.csv"))?; - let factors = read_factors(&path.join("factors.csv"))?; - let factor_texts = read_factor_texts(&path.join("factors.csv"))?; - let candidates = read_candidates(&path.join("candidate_flags.csv"))?; - let benchmarks = read_benchmarks(&path.join("benchmark.csv"))?; - let corporate_actions_path = path.join("corporate_actions.csv"); - let corporate_actions = if corporate_actions_path.exists() { - read_corporate_actions(&corporate_actions_path)? - } else { - Vec::new() - }; - let execution_quotes_path = path.join("execution_quotes.csv"); - let execution_quotes = if execution_quotes_path.exists() { - read_execution_quotes(&execution_quotes_path)? - } else { - Vec::new() - }; - let futures_params_path = path.join("futures_trading_parameters.csv"); - let futures_params = if futures_params_path.exists() { - read_futures_trading_parameters(&futures_params_path)? - } else { - Vec::new() - }; - let order_book_depth_path = path.join("order_book_depth.csv"); - let order_book_depth = if order_book_depth_path.exists() { - read_order_book_depth(&order_book_depth_path)? - } else { - Vec::new() - }; - Self::from_components_with_actions_quotes_futures_depth_and_factor_texts( - instruments, - market, - factors, - candidates, - benchmarks, - corporate_actions, - execution_quotes, - futures_params, - order_book_depth, - factor_texts, - ) - } - - pub fn from_partitioned_dir(path: &Path) -> Result { - let instruments = read_instruments(&path.join("instruments.csv"))?; - let benchmarks = read_partitioned_dir(&path.join("benchmark"), read_benchmarks)?; - let market = read_partitioned_dir(&path.join("market"), read_market)?; - let factors = read_partitioned_dir(&path.join("factors"), read_factors)?; - let factor_texts = read_partitioned_dir(&path.join("factors"), read_factor_texts)?; - let candidates = read_partitioned_dir(&path.join("candidates"), read_candidates)?; - let corporate_actions_dir = path.join("corporate_actions"); - let corporate_actions = if corporate_actions_dir.exists() { - read_partitioned_dir(&corporate_actions_dir, read_corporate_actions)? - } else { - Vec::new() - }; - let execution_quotes_dir = path.join("execution_quotes"); - let execution_quotes = if execution_quotes_dir.exists() { - read_partitioned_dir(&execution_quotes_dir, read_execution_quotes)? - } else { - Vec::new() - }; - let futures_params_dir = path.join("futures_trading_parameters"); - let futures_params = if futures_params_dir.exists() { - read_partitioned_dir(&futures_params_dir, read_futures_trading_parameters)? - } else { - Vec::new() - }; - let order_book_depth_dir = path.join("order_book_depth"); - let order_book_depth = if order_book_depth_dir.exists() { - read_partitioned_dir(&order_book_depth_dir, read_order_book_depth)? - } else { - Vec::new() - }; - Self::from_components_with_actions_quotes_futures_depth_and_factor_texts( - instruments, - market, - factors, - candidates, - benchmarks, - corporate_actions, - execution_quotes, - futures_params, - order_book_depth, - factor_texts, - ) - } - pub fn from_components( instruments: Vec, market: Vec, @@ -2665,157 +2561,6 @@ impl DataSet { } } -fn read_instruments(path: &Path) -> Result, DataSetError> { - let rows = read_rows(path)?; - let mut instruments = Vec::new(); - for row in rows { - instruments.push(Instrument { - symbol: row.get(0)?.to_string(), - name: row.get(1)?.to_string(), - board: row.get(2)?.to_string(), - round_lot: row.parse_optional_u32(3).unwrap_or(100), - listed_at: row.parse_optional_date(4)?, - delisted_at: row.parse_optional_date(5)?, - status: row - .fields - .get(6) - .map(|value| value.trim()) - .filter(|value| !value.is_empty()) - .unwrap_or("active") - .to_string(), - }); - } - Ok(instruments) -} - -fn read_market(path: &Path) -> Result, DataSetError> { - let rows = read_rows(path)?; - let mut snapshots = Vec::new(); - for row in rows { - let open = row.parse_f64(2)?; - let close = row.parse_f64(5)?; - let prev_close = row.parse_f64(6)?; - let price_tick = row.parse_optional_f64(15).unwrap_or(0.01); - let derived_upper_limit = round_price_to_tick(prev_close * 1.10, price_tick); - let derived_lower_limit = round_price_to_tick(prev_close * 0.90, price_tick); - snapshots.push(DailyMarketSnapshot { - date: row.parse_date(0)?, - symbol: row.get(1)?.to_string(), - timestamp: row - .fields - .get(16) - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()), - day_open: row.parse_optional_f64(11).unwrap_or(open), - open, - high: row.parse_f64(3)?, - low: row.parse_f64(4)?, - close, - last_price: row.parse_optional_f64(12).unwrap_or(close), - bid1: row.parse_optional_f64(13).unwrap_or(close), - ask1: row.parse_optional_f64(14).unwrap_or(close), - prev_close, - volume: row.parse_u64(7)?, - minute_volume: row.parse_optional_u64(17).unwrap_or_default(), - bid1_volume: row.parse_optional_u64(18).unwrap_or_default(), - ask1_volume: row.parse_optional_u64(19).unwrap_or_default(), - trading_phase: row - .fields - .get(20) - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()), - paused: row.parse_bool(8)?, - upper_limit: row.parse_optional_f64(9).unwrap_or(derived_upper_limit), - lower_limit: row.parse_optional_f64(10).unwrap_or(derived_lower_limit), - price_tick, - }); - } - Ok(snapshots) -} - -fn read_factors(path: &Path) -> Result, DataSetError> { - let rows = read_rows(path)?; - let mut snapshots = Vec::new(); - for row in rows { - let (extra_factors, _) = parse_extra_factor_maps(&row); - snapshots.push(DailyFactorSnapshot { - date: row.parse_date(0)?, - symbol: row.get(1)?.to_string(), - market_cap_bn: row.parse_f64(2)?, - free_float_cap_bn: row.parse_f64(3)?, - pe_ttm: row.parse_f64(4)?, - turnover_ratio: row.parse_optional_f64(5), - effective_turnover_ratio: row.parse_optional_f64(6), - extra_factors, - }); - } - Ok(snapshots) -} - -fn read_factor_texts(path: &Path) -> Result, DataSetError> { - let rows = read_rows(path)?; - let mut text_values = Vec::new(); - for row in rows { - let date = row.parse_date(0)?; - let symbol = row.get(1)?.to_string(); - let (_, extra_text_factors) = parse_extra_factor_maps(&row); - for (field, value) in extra_text_factors { - text_values.push(FactorTextValue { - date, - symbol: symbol.clone(), - field, - value, - }); - } - } - Ok(text_values) -} - -fn parse_extra_factor_maps(row: &CsvRow) -> (BTreeMap, BTreeMap) { - let mut numeric = BTreeMap::new(); - let mut text = BTreeMap::new(); - for value in row.fields.get(7).into_iter().chain(row.fields.get(8)) { - merge_extra_factor_json(value, &mut numeric, &mut text); - } - (numeric, text) -} - -fn merge_extra_factor_json( - raw: &str, - numeric: &mut BTreeMap, - text: &mut BTreeMap, -) { - let trimmed = raw.trim(); - if trimmed.is_empty() { - return; - } - let Ok(serde_json::Value::Object(map)) = serde_json::from_str::(trimmed) - else { - return; - }; - for (key, value) in map { - let key = normalize_field(&key); - if key.is_empty() { - continue; - } - match value { - serde_json::Value::Number(number) => { - if let Some(value) = number.as_f64().filter(|value| value.is_finite()) { - numeric.insert(key, value); - } - } - serde_json::Value::String(value) => { - text.insert(key, value); - } - serde_json::Value::Bool(value) => { - numeric.insert(key.clone(), if value { 1.0 } else { 0.0 }); - text.insert(key, value.to_string()); - } - _ => {} - } - } -} - fn normalized_aliases(values: &[String]) -> Vec { let mut aliases = Vec::new(); for value in values { @@ -3123,374 +2868,6 @@ fn take_last(mut rows: Vec, count: usize) -> Vec { rows.split_off(rows.len() - count) } -fn read_candidates(path: &Path) -> Result, DataSetError> { - let rows = read_rows(path)?; - let mut snapshots = Vec::new(); - for row in rows { - snapshots.push(CandidateEligibility { - date: row.parse_date(0)?, - symbol: row.get(1)?.to_string(), - is_st: row.parse_bool(2)?, - is_star_st: row.parse_bool(3)?, - is_new_listing: row.parse_bool(4)?, - is_paused: row.parse_bool(5)?, - allow_buy: row.parse_bool(6)?, - allow_sell: row.parse_bool(7)?, - is_kcb: row.parse_optional_bool(8).unwrap_or(false), - is_one_yuan: row.parse_optional_bool(9).unwrap_or(false), - risk_level_code: row - .fields - .get(10) - .map(String::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(ToOwned::to_owned), - }); - } - Ok(snapshots) -} - -fn read_benchmarks(path: &Path) -> Result, DataSetError> { - let rows = read_rows(path)?; - let mut snapshots = Vec::new(); - for row in rows { - snapshots.push(BenchmarkSnapshot { - date: row.parse_date(0)?, - benchmark: row.get(1)?.to_string(), - open: row.parse_f64(2)?, - close: row.parse_f64(3)?, - prev_close: row.parse_f64(4)?, - volume: row.parse_u64(5)?, - }); - } - Ok(snapshots) -} - -fn read_corporate_actions(path: &Path) -> Result, DataSetError> { - let rows = read_rows(path)?; - let mut snapshots = Vec::new(); - for row in rows { - let has_payable_date = row.fields.len() >= 10; - let payable_date = if has_payable_date { - row.parse_optional_date(2)? - } else { - None - }; - let offset = if has_payable_date { 1 } else { 0 }; - snapshots.push(CorporateAction { - date: row.parse_date(0)?, - symbol: row.get(1)?.to_string(), - payable_date, - share_cash: row.parse_optional_f64(2 + offset).unwrap_or(0.0), - share_bonus: row.parse_optional_f64(3 + offset).unwrap_or(0.0), - share_gift: row.parse_optional_f64(4 + offset).unwrap_or(0.0), - issue_quantity: row.parse_optional_f64(5 + offset).unwrap_or(0.0), - issue_price: row.parse_optional_f64(6 + offset).unwrap_or(0.0), - reform: row.parse_optional_bool(7 + offset).unwrap_or(false), - adjust_factor: row.parse_optional_f64(8 + offset), - successor_symbol: row - .fields - .get(9 + offset) - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()), - successor_ratio: row.parse_optional_f64(10 + offset), - successor_cash: row.parse_optional_f64(11 + offset), - }); - } - Ok(snapshots) -} - -fn read_execution_quotes(path: &Path) -> Result, DataSetError> { - let rows = read_rows(path)?; - let mut quotes = Vec::new(); - for row in rows { - quotes.push(IntradayExecutionQuote { - date: row.parse_date(0)?, - symbol: row.get(1)?.to_string(), - timestamp: row.parse_datetime(2)?, - last_price: row.parse_optional_f64(3).unwrap_or_default(), - bid1: row.parse_optional_f64(4).unwrap_or_default(), - ask1: row.parse_optional_f64(5).unwrap_or_default(), - bid1_volume: row.parse_optional_u64(6).unwrap_or_default(), - ask1_volume: row.parse_optional_u64(7).unwrap_or_default(), - volume_delta: row.parse_optional_u64(8).unwrap_or_default(), - amount_delta: row.parse_optional_f64(9).unwrap_or_default(), - trading_phase: row - .fields - .get(10) - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()), - }); - } - Ok(quotes) -} - -fn read_order_book_depth(path: &Path) -> Result, DataSetError> { - let rows = read_rows(path)?; - let mut levels = Vec::new(); - for row in rows { - levels.push(IntradayOrderBookDepthLevel { - date: row.parse_date(0)?, - symbol: row.get(1)?.to_string(), - timestamp: row.parse_datetime(2)?, - level: row - .parse_optional_u32(3) - .unwrap_or(1) - .clamp(1, u8::MAX as u32) as u8, - bid_price: row.parse_optional_f64(4).unwrap_or_default(), - bid_volume: row.parse_optional_u64(5).unwrap_or_default(), - ask_price: row.parse_optional_f64(6).unwrap_or_default(), - ask_volume: row.parse_optional_u64(7).unwrap_or_default(), - }); - } - Ok(levels) -} - -fn read_futures_trading_parameters( - path: &Path, -) -> Result, DataSetError> { - let rows = read_rows(path)?; - let mut params = Vec::new(); - for row in rows { - let first = row.get(0)?.trim(); - let (effective_date, symbol_index) = if NaiveDate::parse_from_str(first, "%Y-%m-%d").is_ok() - { - (row.parse_optional_date(0)?, 1) - } else { - (None, 0) - }; - params.push(FuturesTradingParameter { - effective_date, - symbol: row.get(symbol_index)?.to_string(), - contract_multiplier: row.parse_optional_f64(symbol_index + 1).unwrap_or(1.0), - long_margin_rate: row.parse_optional_f64(symbol_index + 2).unwrap_or(0.0), - short_margin_rate: row.parse_optional_f64(symbol_index + 3).unwrap_or(0.0), - commission_type: row - .fields - .get(symbol_index + 4) - .map(|value| FuturesCommissionType::parse(value)) - .unwrap_or(FuturesCommissionType::ByMoney), - open_commission_ratio: row.parse_optional_f64(symbol_index + 5).unwrap_or(0.0), - close_commission_ratio: row.parse_optional_f64(symbol_index + 6).unwrap_or(0.0), - close_today_commission_ratio: row - .parse_optional_f64(symbol_index + 7) - .unwrap_or_else(|| row.parse_optional_f64(symbol_index + 6).unwrap_or(0.0)), - price_tick: row.parse_optional_f64(symbol_index + 8).unwrap_or(1.0), - }); - } - Ok(params) -} - -struct CsvRow { - path: String, - line: usize, - fields: Vec, -} - -impl CsvRow { - fn get(&self, index: usize) -> Result<&str, DataSetError> { - self.fields - .get(index) - .map(String::as_str) - .ok_or_else(|| DataSetError::InvalidRow { - path: self.path.clone(), - line: self.line, - message: format!("missing column {index}"), - }) - } - - fn parse_date(&self, index: usize) -> Result { - NaiveDate::parse_from_str(self.get(index)?, "%Y-%m-%d").map_err(|err| { - DataSetError::InvalidRow { - path: self.path.clone(), - line: self.line, - message: format!("invalid date: {err}"), - } - }) - } - - fn parse_f64(&self, index: usize) -> Result { - self.get(index)? - .parse::() - .map_err(|err| DataSetError::InvalidRow { - path: self.path.clone(), - line: self.line, - message: format!("invalid f64: {err}"), - }) - } - - fn parse_u64(&self, index: usize) -> Result { - self.get(index)? - .parse::() - .map_err(|err| DataSetError::InvalidRow { - path: self.path.clone(), - line: self.line, - message: format!("invalid u64: {err}"), - }) - } - - fn parse_optional_f64(&self, index: usize) -> Option { - self.fields.get(index).and_then(|value| { - let trimmed = value.trim(); - if trimmed.is_empty() { - None - } else { - trimmed.parse::().ok() - } - }) - } - - fn parse_bool(&self, index: usize) -> Result { - self.get(index)? - .parse::() - .map_err(|err| DataSetError::InvalidRow { - path: self.path.clone(), - line: self.line, - message: format!("invalid bool: {err}"), - }) - } - - fn parse_optional_bool(&self, index: usize) -> Option { - self.fields - .get(index) - .and_then(|value| value.parse::().ok()) - } - - fn parse_optional_date(&self, index: usize) -> Result, DataSetError> { - let Some(value) = self.fields.get(index) else { - return Ok(None); - }; - let trimmed = value.trim(); - if trimmed.is_empty() { - return Ok(None); - } - NaiveDate::parse_from_str(trimmed, "%Y-%m-%d") - .map(Some) - .map_err(|err| DataSetError::InvalidRow { - path: self.path.clone(), - line: self.line, - message: format!("invalid optional date: {err}"), - }) - } - - fn parse_datetime(&self, index: usize) -> Result { - NaiveDateTime::parse_from_str(self.get(index)?, "%Y-%m-%d %H:%M:%S").map_err(|err| { - DataSetError::InvalidRow { - path: self.path.clone(), - line: self.line, - message: format!("invalid datetime: {err}"), - } - }) - } - - fn parse_optional_u32(&self, index: usize) -> Option { - self.fields.get(index).and_then(|value| { - let trimmed = value.trim(); - if trimmed.is_empty() { - None - } else { - trimmed.parse::().ok() - } - }) - } - - fn parse_optional_u64(&self, index: usize) -> Option { - self.fields.get(index).and_then(|value| { - let trimmed = value.trim(); - if trimmed.is_empty() { - None - } else { - trimmed.parse::().ok() - } - }) - } -} - -fn read_partitioned_dir(dir: &Path, mut loader: F) -> Result, DataSetError> -where - F: FnMut(&Path) -> Result, DataSetError>, -{ - let mut rows = Vec::new(); - let mut stack = vec![dir.to_path_buf()]; - - while let Some(current_dir) = stack.pop() { - let mut entries = fs::read_dir(¤t_dir) - .map_err(|source| DataSetError::Io { - path: current_dir.display().to_string(), - source, - })? - .collect::, _>>() - .map_err(|source| DataSetError::Io { - path: current_dir.display().to_string(), - source, - })?; - entries.sort_by_key(|entry| entry.path()); - - for entry in entries.into_iter().rev() { - let path = entry.path(); - if path.is_dir() { - stack.push(path); - continue; - } - if path.extension().and_then(|x| x.to_str()) != Some("csv") { - continue; - } - rows.extend(loader(&path)?); - } - } - - Ok(rows) -} - -fn read_rows(path: &Path) -> Result, DataSetError> { - let content = fs::read_to_string(path).map_err(|source| DataSetError::Io { - path: path.display().to_string(), - source, - })?; - - let mut rows = Vec::new(); - for (line_idx, line) in content.lines().enumerate() { - let line_no = line_idx + 1; - if line_no == 1 || line.trim().is_empty() { - continue; - } - - rows.push(CsvRow { - path: path.display().to_string(), - line: line_no, - fields: split_csv_line(line), - }); - } - - Ok(rows) -} - -fn split_csv_line(line: &str) -> Vec { - let mut fields = Vec::new(); - let mut field = String::new(); - let mut chars = line.trim_start_matches('\u{feff}').chars().peekable(); - let mut in_quotes = false; - - while let Some(ch) = chars.next() { - match ch { - '"' if in_quotes && chars.peek() == Some(&'"') => { - field.push('"'); - chars.next(); - } - '"' => { - in_quotes = !in_quotes; - } - ',' if !in_quotes => { - fields.push(field.trim().to_string()); - field.clear(); - } - _ => field.push(ch), - } - } - fields.push(field.trim().to_string()); - fields -} - fn group_by_date(rows: Vec, mut date_of: F) -> BTreeMap> where F: FnMut(&T) -> NaiveDate, @@ -3549,15 +2926,6 @@ fn collect_benchmark_code(benchmarks: &[BenchmarkSnapshot]) -> Result f64 { - let effective_tick = if tick.is_finite() && tick > 0.0 { - tick - } else { - 0.01 - }; - ((value / effective_tick).round() * effective_tick * 10000.0).round() / 10000.0 -} - fn prefix_sums(values: &[f64]) -> Vec { let mut prefix = Vec::with_capacity(values.len() + 1); prefix.push(0.0); @@ -3770,15 +3138,6 @@ fn instrument_passes_baseline_selection(instrument: Option<&Instrument>, date: N #[cfg(test)] mod tests { use super::*; - use std::time::{SystemTime, UNIX_EPOCH}; - - fn temp_csv_path(name: &str) -> std::path::PathBuf { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - std::env::temp_dir().join(format!("{}_{}_{}.csv", name, std::process::id(), nanos)) - } fn market_row(date: &str, prev_close: f64, volume: u64) -> DailyMarketSnapshot { DailyMarketSnapshot { @@ -4193,39 +3552,4 @@ mod tests { Some((200.0 + 9_999.0) / 2.0) ); } - - #[test] - fn reads_mixed_numeric_and_text_extra_factors_from_quoted_csv_json() { - let path = temp_csv_path("mixed_factor_maps"); - fs::write( - &path, - concat!( - "date,symbol,market_cap_bn,free_float_cap_bn,pe_ttm,turnover_ratio,effective_turnover_ratio,extra_factors\n", - "2025-01-02,000001.SZ,12,10,8,1,1,\"{\"\"custom_alpha\"\":7,\"\"industry_name\"\":\"\"electronics,hardware\"\",\"\"flag\"\":true}\"\n" - ), - ) - .unwrap(); - - let factors = read_factors(&path).unwrap(); - let text_factors = read_factor_texts(&path).unwrap(); - fs::remove_file(&path).ok(); - - assert_eq!(factors.len(), 1); - assert_eq!( - factors[0].extra_factors.get("custom_alpha").copied(), - Some(7.0) - ); - assert_eq!(factors[0].extra_factors.get("flag").copied(), Some(1.0)); - assert_eq!(text_factors.len(), 2); - assert!( - text_factors - .iter() - .any(|row| row.field == "industry_name" && row.value == "electronics,hardware") - ); - assert!( - text_factors - .iter() - .any(|row| row.field == "flag" && row.value == "true") - ); - } } diff --git a/crates/fidc-core/tests/partitioned_loader.rs b/crates/fidc-core/tests/partitioned_loader.rs deleted file mode 100644 index 2228296..0000000 --- a/crates/fidc-core/tests/partitioned_loader.rs +++ /dev/null @@ -1,93 +0,0 @@ -use fidc_core::DataSet; -use std::fs; -use std::path::PathBuf; -use std::time::{SystemTime, UNIX_EPOCH}; - -fn temp_dir() -> PathBuf { - let uniq = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("clock") - .as_nanos(); - let dir = std::env::temp_dir().join(format!("fidc-bt-partitioned-{uniq}")); - fs::create_dir_all(&dir).expect("mkdir temp"); - dir -} - -#[test] -fn can_load_partitioned_snapshot_dir() { - let dir = temp_dir(); - fs::create_dir_all(dir.join("benchmark/2024/01")).unwrap(); - fs::create_dir_all(dir.join("market/2024/01")).unwrap(); - fs::create_dir_all(dir.join("factors/2024/01")).unwrap(); - fs::create_dir_all(dir.join("candidates/2024/01")).unwrap(); - fs::create_dir_all(dir.join("corporate_actions/2024/01")).unwrap(); - - fs::write( - dir.join("instruments.csv"), - "symbol,name,board,round_lot,listed_at,delisted_at,status\n000001.SZ,PingAn,SZ,100,2020-01-01,,active\n", - ) - .unwrap(); - fs::write( - dir.join("benchmark/2024/01/2024-01-02.csv"), - "date,benchmark,open,close,prev_close,volume\n2024-01-02,CSI300.DEMO,2990,3000,2980,100000000\n", - ) - .unwrap(); - fs::write( - dir.join("market/2024/01/2024-01-02.csv"), - "date,symbol,open,high,low,close,prev_close,volume,paused,upper_limit,lower_limit,day_open,last_price,bid1,ask1,price_tick\n2024-01-02,000001.SZ,10,10.5,9.9,10.2,10,100000,false,11,9,10.1,10.15,10.14,10.16,0.01\n", - ) - .unwrap(); - fs::write( - dir.join("factors/2024/01/2024-01-02.csv"), - "date,symbol,market_cap_bn,free_float_cap_bn,pe_ttm,turnover_ratio,effective_turnover_ratio\n2024-01-02,000001.SZ,40,35,12,3.2,2.1\n", - ) - .unwrap(); - fs::write( - dir.join("candidates/2024/01/2024-01-02.csv"), - "date,symbol,is_st,is_new_listing,is_paused,allow_buy,allow_sell,is_kcb,is_one_yuan\n2024-01-02,000001.SZ,false,false,false,true,true,false,false\n", - ) - .unwrap(); - fs::write( - dir.join("corporate_actions/2024/01/2024-01-02.csv"), - "date,symbol,payable_date,share_cash,share_bonus,share_gift,issue_quantity,issue_price,reform,adjust_factor\n2024-01-02,000001.SZ,2024-01-05,0.5,0.1,0.0,0,0,false,1.05\n", - ) - .unwrap(); - - let data = DataSet::from_partitioned_dir(&dir).expect("partitioned dataset"); - assert_eq!(data.benchmark_code(), "CSI300.DEMO"); - assert!( - data.market_snapshots_on(chrono::NaiveDate::from_ymd_opt(2024, 1, 2).unwrap()) - .len() - == 1 - ); - let market_rows = - data.market_snapshots_on(chrono::NaiveDate::from_ymd_opt(2024, 1, 2).unwrap()); - let snapshot = market_rows.first().expect("market snapshot"); - assert_eq!(snapshot.day_open, 10.1); - assert_eq!(snapshot.last_price, 10.15); - assert_eq!(snapshot.price_tick, 0.01); - assert_eq!( - data.instruments() - .get("000001.SZ") - .expect("instrument") - .round_lot, - 100 - ); - assert_eq!( - data.instruments() - .get("000001.SZ") - .expect("instrument") - .listed_at, - Some(chrono::NaiveDate::from_ymd_opt(2020, 1, 1).unwrap()) - ); - let actions = data.corporate_actions_on(chrono::NaiveDate::from_ymd_opt(2024, 1, 2).unwrap()); - assert_eq!(actions.len(), 1); - assert_eq!( - actions[0].payable_date, - Some(chrono::NaiveDate::from_ymd_opt(2024, 1, 5).unwrap()) - ); - assert!((actions[0].share_cash - 0.5).abs() < 1e-9); - assert!((actions[0].split_ratio() - 1.1).abs() < 1e-9); - - let _ = fs::remove_dir_all(&dir); -} diff --git a/crates/fidc-core/tests/strategy_selection.rs b/crates/fidc-core/tests/strategy_selection.rs index 0c8283b..72a0c2c 100644 --- a/crates/fidc-core/tests/strategy_selection.rs +++ b/crates/fidc-core/tests/strategy_selection.rs @@ -1,15 +1,582 @@ use chrono::NaiveDate; use fidc_core::{ - CnSmallCapRotationConfig, CnSmallCapRotationStrategy, DataSet, OmniMicroCapConfig, + BenchmarkSnapshot, CandidateEligibility, CnSmallCapRotationConfig, CnSmallCapRotationStrategy, + DailyFactorSnapshot, DailyMarketSnapshot, DataSet, Instrument, OmniMicroCapConfig, OmniMicroCapStrategy, PortfolioState, Strategy, StrategyContext, }; use std::collections::BTreeSet; -use std::path::PathBuf; + +fn d(value: &str) -> NaiveDate { + NaiveDate::parse_from_str(value, "%Y-%m-%d").unwrap() +} + +fn instrument(symbol: &str, name: &str) -> Instrument { + Instrument { + symbol: symbol.to_string(), + name: name.to_string(), + board: "Main".to_string(), + round_lot: 100, + listed_at: None, + delisted_at: None, + status: "active".to_string(), + } +} + +fn market( + date: &str, + symbol: &str, + open: f64, + high: f64, + low: f64, + close: f64, + prev_close: f64, + volume: u64, + paused: bool, +) -> DailyMarketSnapshot { + DailyMarketSnapshot { + date: d(date), + symbol: symbol.to_string(), + timestamp: None, + day_open: open, + open, + high, + low, + close, + last_price: close, + bid1: close, + ask1: close, + prev_close, + volume, + minute_volume: 0, + bid1_volume: 0, + ask1_volume: 0, + trading_phase: None, + paused, + upper_limit: (prev_close * 1.10 * 100.0).round() / 100.0, + lower_limit: (prev_close * 0.90 * 100.0).round() / 100.0, + price_tick: 0.01, + } +} + +fn factor( + date: &str, + symbol: &str, + market_cap_bn: f64, + free_float_cap_bn: f64, +) -> DailyFactorSnapshot { + DailyFactorSnapshot { + date: d(date), + symbol: symbol.to_string(), + market_cap_bn, + free_float_cap_bn, + pe_ttm: 18.0, + turnover_ratio: None, + effective_turnover_ratio: None, + extra_factors: Default::default(), + } +} + +fn candidate( + date: &str, + symbol: &str, + is_new_listing: bool, + is_paused: bool, + allow_buy: bool, + allow_sell: bool, +) -> CandidateEligibility { + CandidateEligibility { + date: d(date), + symbol: symbol.to_string(), + is_st: false, + is_star_st: false, + is_new_listing, + is_paused, + allow_buy, + allow_sell, + is_kcb: false, + is_one_yuan: false, + risk_level_code: None, + } +} + +fn benchmark(date: &str, open: f64, close: f64, prev_close: f64, volume: u64) -> BenchmarkSnapshot { + BenchmarkSnapshot { + date: d(date), + benchmark: "CSI300.DEMO".to_string(), + open, + close, + prev_close, + volume, + } +} + +fn strategy_test_dataset() -> DataSet { + let dates = [ + "2024-01-02", + "2024-01-03", + "2024-01-04", + "2024-01-05", + "2024-01-08", + "2024-01-09", + "2024-01-10", + "2024-01-11", + "2024-01-12", + ]; + let instruments = vec![ + instrument("000001.SZ", "Alpha Components"), + instrument("000002.SZ", "Beta Precision"), + instrument("000003.SZ", "Charlie Materials"), + instrument("600001.SH", "Delta Industrials"), + ]; + let market = vec![ + market( + "2024-01-02", + "000001.SZ", + 10.0, + 10.2, + 9.9, + 10.1, + 9.8, + 1_200_000, + false, + ), + market( + "2024-01-02", + "000002.SZ", + 11.0, + 11.3, + 10.9, + 11.2, + 10.8, + 1_100_000, + false, + ), + market( + "2024-01-02", + "000003.SZ", + 8.0, + 8.1, + 7.8, + 7.9, + 8.0, + 900_000, + false, + ), + market( + "2024-01-02", + "600001.SH", + 15.0, + 15.2, + 14.9, + 15.1, + 15.0, + 800_000, + false, + ), + market( + "2024-01-03", + "000001.SZ", + 10.2, + 10.5, + 10.1, + 10.4, + 10.1, + 1_250_000, + false, + ), + market( + "2024-01-03", + "000002.SZ", + 11.2, + 11.6, + 11.1, + 11.5, + 11.2, + 1_120_000, + false, + ), + market( + "2024-01-03", + "000003.SZ", + 7.8, + 7.9, + 7.3, + 7.4, + 7.9, + 930_000, + false, + ), + market( + "2024-01-03", + "600001.SH", + 15.1, + 15.3, + 15.0, + 15.2, + 15.1, + 820_000, + false, + ), + market( + "2024-01-04", + "000001.SZ", + 10.5, + 10.8, + 10.4, + 10.7, + 10.4, + 1_280_000, + false, + ), + market( + "2024-01-04", + "000002.SZ", + 11.4, + 11.9, + 11.3, + 11.8, + 11.5, + 1_150_000, + false, + ), + market( + "2024-01-04", + "000003.SZ", + 7.3, + 7.4, + 7.0, + 7.1, + 7.4, + 940_000, + false, + ), + market( + "2024-01-04", + "600001.SH", + 15.2, + 15.5, + 15.1, + 15.4, + 15.2, + 830_000, + false, + ), + market( + "2024-01-05", + "000001.SZ", + 10.8, + 11.1, + 10.7, + 11.0, + 10.7, + 1_300_000, + false, + ), + market( + "2024-01-05", + "000002.SZ", + 11.9, + 12.1, + 11.8, + 12.0, + 11.8, + 1_180_000, + false, + ), + market( + "2024-01-05", + "000003.SZ", + 7.0, + 7.1, + 6.8, + 6.9, + 7.1, + 950_000, + false, + ), + market( + "2024-01-05", + "600001.SH", + 15.4, + 15.6, + 15.3, + 15.5, + 15.4, + 840_000, + false, + ), + market( + "2024-01-08", + "000001.SZ", + 11.1, + 11.6, + 11.0, + 11.5, + 11.0, + 1_400_000, + false, + ), + market( + "2024-01-08", + "000002.SZ", + 12.1, + 12.5, + 12.0, + 12.4, + 12.0, + 1_200_000, + false, + ), + market( + "2024-01-08", + "000003.SZ", + 7.0, + 7.3, + 6.9, + 7.2, + 6.9, + 980_000, + false, + ), + market( + "2024-01-08", + "600001.SH", + 15.5, + 15.7, + 15.4, + 15.6, + 15.5, + 850_000, + false, + ), + market( + "2024-01-09", + "000001.SZ", + 11.6, + 12.4, + 11.5, + 12.3, + 11.5, + 1_500_000, + false, + ), + market( + "2024-01-09", + "000002.SZ", + 12.5, + 12.9, + 12.4, + 12.8, + 12.4, + 1_250_000, + false, + ), + market( + "2024-01-09", + "000003.SZ", + 7.2, + 7.5, + 7.1, + 7.4, + 7.2, + 990_000, + false, + ), + market( + "2024-01-09", + "600001.SH", + 15.6, + 15.7, + 15.4, + 15.5, + 15.6, + 860_000, + false, + ), + market( + "2024-01-10", + "000001.SZ", + 12.2, + 12.3, + 11.9, + 12.0, + 12.3, + 1_450_000, + false, + ), + market( + "2024-01-10", + "000002.SZ", + 12.7, + 12.8, + 12.5, + 12.6, + 12.8, + 1_220_000, + false, + ), + market( + "2024-01-10", + "000003.SZ", + 7.5, + 7.6, + 7.4, + 7.5, + 7.4, + 1_000_000, + false, + ), + market( + "2024-01-10", + "600001.SH", + 15.4, + 15.5, + 15.1, + 15.2, + 15.5, + 870_000, + false, + ), + market( + "2024-01-11", + "000001.SZ", + 12.0, + 12.1, + 11.5, + 11.6, + 12.0, + 1_420_000, + false, + ), + market( + "2024-01-11", + "000002.SZ", + 12.5, + 12.6, + 12.1, + 12.2, + 12.6, + 1_210_000, + false, + ), + market( + "2024-01-11", + "000003.SZ", + 7.4, + 7.5, + 7.2, + 7.3, + 7.5, + 980_000, + false, + ), + market( + "2024-01-11", + "600001.SH", + 15.2, + 15.2, + 15.2, + 15.2, + 15.2, + 0, + true, + ), + market( + "2024-01-12", + "000001.SZ", + 11.5, + 11.6, + 11.1, + 11.2, + 11.6, + 1_380_000, + false, + ), + market( + "2024-01-12", + "000002.SZ", + 12.1, + 12.2, + 11.8, + 11.9, + 12.2, + 1_190_000, + false, + ), + market( + "2024-01-12", + "000003.SZ", + 7.2, + 7.2, + 6.9, + 7.0, + 7.3, + 960_000, + false, + ), + market( + "2024-01-12", + "600001.SH", + 14.8, + 15.0, + 14.7, + 14.9, + 15.2, + 850_000, + false, + ), + ]; + let factors = dates + .iter() + .enumerate() + .flat_map(|(idx, date)| { + let i = idx as f64; + [ + factor(date, "000001.SZ", 38.0 + i, 24.0 + i * 0.5), + factor(date, "000002.SZ", 45.0 + i, 30.0 + i * 0.5), + factor(date, "000003.SZ", 65.0 - i, 40.0 - i * 0.5), + factor(date, "600001.SH", 85.0 + i, 55.0 + i * 0.5), + ] + }) + .collect::>(); + let candidates = dates + .iter() + .flat_map(|date| { + let first_two = *date == "2024-01-02" || *date == "2024-01-03"; + let paused_600001 = *date == "2024-01-11"; + [ + candidate(date, "000001.SZ", first_two, false, !first_two, true), + candidate(date, "000002.SZ", false, false, true, true), + candidate(date, "000003.SZ", false, false, true, true), + candidate( + date, + "600001.SH", + false, + paused_600001, + !paused_600001, + !paused_600001, + ), + ] + }) + .collect::>(); + let benchmarks = vec![ + benchmark("2024-01-02", 2990.0, 3000.0, 2980.0, 100_000_000), + benchmark("2024-01-03", 3005.0, 3020.0, 3000.0, 102_000_000), + benchmark("2024-01-04", 3025.0, 3050.0, 3020.0, 105_000_000), + benchmark("2024-01-05", 3055.0, 3080.0, 3050.0, 108_000_000), + benchmark("2024-01-08", 3085.0, 3110.0, 3080.0, 109_000_000), + benchmark("2024-01-09", 3100.0, 3090.0, 3110.0, 107_000_000), + benchmark("2024-01-10", 3080.0, 3040.0, 3090.0, 111_000_000), + benchmark("2024-01-11", 3030.0, 2990.0, 3040.0, 115_000_000), + benchmark("2024-01-12", 2980.0, 2950.0, 2990.0, 118_000_000), + ]; + DataSet::from_components(instruments, market, factors, candidates, benchmarks) + .expect("strategy test dataset") +} #[test] fn strategy_emits_target_weights_and_diagnostics() { - let data_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../data/demo"); - let data = DataSet::from_csv_dir(&data_dir).expect("demo data"); + let data = strategy_test_dataset(); let decision_date = NaiveDate::from_ymd_opt(2024, 1, 10).unwrap(); let execution_date = NaiveDate::from_ymd_opt(2024, 1, 11).unwrap(); let portfolio = PortfolioState::new(1_000_000.0); @@ -53,8 +620,7 @@ fn strategy_emits_target_weights_and_diagnostics() { #[test] fn omni_strategy_emits_same_day_decision() { - let data_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../data/demo"); - let data = DataSet::from_csv_dir(&data_dir).expect("demo data"); + let data = strategy_test_dataset(); let execution_date = NaiveDate::from_ymd_opt(2024, 1, 10).unwrap(); let portfolio = PortfolioState::new(1_000_000.0); let mut cfg = OmniMicroCapConfig::omni_microcap(); diff --git a/data/demo/000852量价微盘平台策略.engine-script.txt b/data/demo/000852量价微盘平台策略.engine-script.txt deleted file mode 100644 index 8b66d65..0000000 --- a/data/demo/000852量价微盘平台策略.engine-script.txt +++ /dev/null @@ -1,63 +0,0 @@ -let refresh_rate = 15; -let stocknum = 40; -let close_rate = 1.07; -let loss_rate = 0.93; -let rsi_rate = 1.0001; -let trade_rate = 0.5; -let xs = 4 / 500; -let base_index_level = 2000; -let base_cap_floor = 3; -let base_cap_ceiling = 28; - -fn band_start(current_price, base_index_level, xs, base_cap_floor) { - if current_price == base_index_level { - base_cap_floor - } else if current_price > 0 { - round((current_price - base_index_level) * xs + base_cap_floor) - } else { - base_cap_floor - } -} - -fn band_end(current_price, base_index_level, xs, base_cap_ceiling) { - if current_price == base_index_level { - base_cap_ceiling - } else if current_price > 0 { - round((current_price - base_index_level) * xs + base_cap_ceiling) - } else { - base_cap_ceiling - } -} - -strategy("microcap_volume_trend_000852") { - market("CN_A") - benchmark("000852.SH") - signal("000852.SH") - - rebalance.every_days(refresh_rate).at("10:18") - - universe.exclude("paused", "st", "kcb", "one_yuan", "new_listing") - - selection.limit(stocknum) - selection.market_cap_band( - field="market_cap", - lower=band_start(signal_close, base_index_level, xs, base_cap_floor), - upper=band_end(signal_close, base_index_level, xs, base_cap_ceiling) - ) - - risk.index_exposure( - signal_ma5 > signal_ma10 * rsi_rate ? 1.0 : trade_rate - ) - - filter.stock_expr( - stock_ma5 > stock_ma10 * rsi_rate && - stock_ma10 > stock_ma30 * rsi_rate && - rolling_mean("volume", 5) < rolling_mean("volume", 60) - ) - - risk.take_profit(close_rate) - risk.stop_loss(loss_rate) - allocation.buy_scale(touched_upper_limit ? 1.0 : trade_rate) - - ordering.rank_by("market_cap", "asc") -} diff --git a/data/demo/000852量价微盘平台策略.strategy_spec.json b/data/demo/000852量价微盘平台策略.strategy_spec.json deleted file mode 100644 index a25aa0f..0000000 --- a/data/demo/000852量价微盘平台策略.strategy_spec.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "strategyId": "microcap_volume_trend_000852", - "version": "2", - "parser": "omniquant-engine-script-v2", - "market": "CN_A", - "signalSymbol": "000852.SH", - "benchmark": { - "instrumentId": "000852.SH", - "fallbackInstrumentId": "000852.SH" - }, - "engineConfig": { - "market": "CN_A", - "signalSymbol": "000852.SH", - "benchmarkSymbol": "000852.SH", - "refreshRate": 15, - "rankLimit": 40 - }, - "runtimeExpressions": { - "prelude": "let refresh_rate = 15;\nlet stocknum = 40;\nlet close_rate = 1.07;\nlet loss_rate = 0.93;\nlet rsi_rate = 1.0001;\nlet trade_rate = 0.5;\nlet xs = 4 / 500;\nlet base_index_level = 2000;\nlet base_cap_floor = 3;\nlet base_cap_ceiling = 28;\nfn band_start(current_price, base_index_level, xs, base_cap_floor) {\n if current_price == base_index_level {\n base_cap_floor\n } else if current_price > 0 {\n round((current_price - base_index_level) * xs + base_cap_floor)\n } else {\n base_cap_floor\n }\n}\nfn band_end(current_price, base_index_level, xs, base_cap_ceiling) {\n if current_price == base_index_level {\n base_cap_ceiling\n } else if current_price > 0 {\n round((current_price - base_index_level) * xs + base_cap_ceiling)\n } else {\n base_cap_ceiling\n }\n}", - "selection": { - "limitExpr": "stocknum", - "marketCapField": "market_cap", - "marketCapLowerExpr": "band_start(signal_close, base_index_level, xs, base_cap_floor)", - "marketCapUpperExpr": "band_end(signal_close, base_index_level, xs, base_cap_ceiling)", - "stockFilterExpr": "stock_ma5 > stock_ma10 * rsi_rate && stock_ma10 > stock_ma30 * rsi_rate && rolling_mean(\"volume\", 5) < rolling_mean(\"volume\", 60)" - }, - "risk": { - "exposureExpr": "signal_ma5 > signal_ma10 * rsi_rate ? 1.0 : trade_rate", - "stopLossExpr": "loss_rate", - "takeProfitExpr": "close_rate" - }, - "allocation": { - "buyScaleExpr": "touched_upper_limit ? 1.0 : trade_rate" - }, - "ordering": { - "rankBy": "market_cap", - "rankExpr": "", - "rankOrder": "asc" - } - } -} diff --git a/data/demo/ai_generated_000001_open_cap_band.engine-script.txt b/data/demo/ai_generated_000001_open_cap_band.engine-script.txt deleted file mode 100644 index 9012b13..0000000 --- a/data/demo/ai_generated_000001_open_cap_band.engine-script.txt +++ /dev/null @@ -1,42 +0,0 @@ -let refresh_rate = 15; -let stocknum = 40; -let xs = 0.008; -let base_index_level = 2000; -let lower_offset = 3; -let upper_offset = 28; - -fn cap_floor(current_price, base_index_level, xs, lower_offset) { -round((current_price - base_index_level) * xs + lower_offset) -} - -fn cap_ceiling(current_price, base_index_level, xs, upper_offset) { -round((current_price - base_index_level) * xs + upper_offset) -} - -strategy("ai_generated_000001_open_cap_band") { -market("CN_A") -benchmark("000852.SH") -signal("000001.SH") - -rebalance.every_days(refresh_rate).at("10:18") - -universe.exclude("paused", "st", "kcb", "one_yuan", "new_listing") - -selection.limit(stocknum) -selection.market_cap_band( -field="market_cap", -lower=cap_floor(signal_open, base_index_level, xs, lower_offset), -upper=cap_ceiling(signal_open, base_index_level, xs, upper_offset) -) - -filter.stock_expr( -stock_ma5 > stock_ma10 && -stock_ma10 > stock_ma30 && -rolling_mean("volume", 5) < rolling_mean("volume", 60) && -!ends_with(symbol, ".BJ") && -!at_upper_limit && -!at_lower_limit -) - - ordering.rank_by("market_cap", "asc") -} diff --git a/data/demo/ai_generated_000001_open_cap_band.strategy_spec.json b/data/demo/ai_generated_000001_open_cap_band.strategy_spec.json deleted file mode 100644 index 00f4e7d..0000000 --- a/data/demo/ai_generated_000001_open_cap_band.strategy_spec.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "strategyId": "ai_generated_000001_open_cap_band", - "version": "2", - "parser": "omniquant-engine-script-v2", - "market": "CN_A", - "signalSymbol": "000001.SH", - "benchmark": { - "instrumentId": "000852.SH", - "fallbackInstrumentId": "000852.SH" - }, - "engineConfig": { - "market": "CN_A", - "signalSymbol": "000001.SH", - "benchmarkSymbol": "000852.SH", - "refreshRate": 15, - "rankLimit": 40 - }, - "runtimeExpressions": { - "prelude": "let refresh_rate = 15;\nlet stocknum = 40;\nlet xs = 0.008;\nlet base_index_level = 2000;\nlet lower_offset = 3;\nlet upper_offset = 28;\n\nfn cap_floor(current_price, base_index_level, xs, lower_offset) {\nround((current_price - base_index_level) * xs + lower_offset)\n}\n\nfn cap_ceiling(current_price, base_index_level, xs, upper_offset) {\nround((current_price - base_index_level) * xs + upper_offset)\n}", - "selection": { - "limitExpr": "stocknum", - "marketCapField": "market_cap", - "marketCapLowerExpr": "cap_floor(signal_open, base_index_level, xs, lower_offset)", - "marketCapUpperExpr": "cap_ceiling(signal_open, base_index_level, xs, upper_offset)", - "stockFilterExpr": "stock_ma5 > stock_ma10 && stock_ma10 > stock_ma30 && rolling_mean(\"volume\", 5) < rolling_mean(\"volume\", 60) && !ends_with(symbol, \".BJ\") && !at_upper_limit && !at_lower_limit" - }, - "ordering": { - "rankBy": "market_cap", - "rankExpr": "", - "rankOrder": "asc" - } - } -} diff --git a/data/demo/benchmark.csv b/data/demo/benchmark.csv deleted file mode 100644 index df11a3c..0000000 --- a/data/demo/benchmark.csv +++ /dev/null @@ -1,10 +0,0 @@ -date,benchmark,open,close,prev_close,volume -2024-01-02,CSI300.DEMO,2990,3000,2980,100000000 -2024-01-03,CSI300.DEMO,3005,3020,3000,102000000 -2024-01-04,CSI300.DEMO,3025,3050,3020,105000000 -2024-01-05,CSI300.DEMO,3055,3080,3050,108000000 -2024-01-08,CSI300.DEMO,3085,3110,3080,109000000 -2024-01-09,CSI300.DEMO,3100,3090,3110,107000000 -2024-01-10,CSI300.DEMO,3080,3040,3090,111000000 -2024-01-11,CSI300.DEMO,3030,2990,3040,115000000 -2024-01-12,CSI300.DEMO,2980,2950,2990,118000000 diff --git a/data/demo/candidate_flags.csv b/data/demo/candidate_flags.csv deleted file mode 100644 index f062102..0000000 --- a/data/demo/candidate_flags.csv +++ /dev/null @@ -1,37 +0,0 @@ -date,symbol,is_st,is_new_listing,is_paused,allow_buy,allow_sell,is_kcb,is_one_yuan -2024-01-02,000001.SZ,false,true,false,false,true,false,false -2024-01-02,000002.SZ,false,false,false,true,true,false,false -2024-01-02,000003.SZ,false,false,false,true,true,false,false -2024-01-02,600001.SH,false,false,false,true,true,false,false -2024-01-03,000001.SZ,false,true,false,false,true,false,false -2024-01-03,000002.SZ,false,false,false,true,true,false,false -2024-01-03,000003.SZ,false,false,false,true,true,false,false -2024-01-03,600001.SH,false,false,false,true,true,false,false -2024-01-04,000001.SZ,false,false,false,true,true,false,false -2024-01-04,000002.SZ,false,false,false,true,true,false,false -2024-01-04,000003.SZ,false,false,false,true,true,false,false -2024-01-04,600001.SH,false,false,false,true,true,false,false -2024-01-05,000001.SZ,false,false,false,true,true,false,false -2024-01-05,000002.SZ,false,false,false,true,true,false,false -2024-01-05,000003.SZ,false,false,false,true,true,false,false -2024-01-05,600001.SH,false,false,false,true,true,false,false -2024-01-08,000001.SZ,false,false,false,true,true,false,false -2024-01-08,000002.SZ,false,false,false,true,true,false,false -2024-01-08,000003.SZ,false,false,false,true,true,false,false -2024-01-08,600001.SH,false,false,false,true,true,false,false -2024-01-09,000001.SZ,false,false,false,true,true,false,false -2024-01-09,000002.SZ,false,false,false,true,true,false,false -2024-01-09,000003.SZ,false,false,false,true,true,false,false -2024-01-09,600001.SH,false,false,false,true,true,false,false -2024-01-10,000001.SZ,false,false,false,true,true,false,false -2024-01-10,000002.SZ,false,false,false,true,true,false,false -2024-01-10,000003.SZ,false,false,false,true,true,false,false -2024-01-10,600001.SH,false,false,false,true,true,false,false -2024-01-11,000001.SZ,false,false,false,true,true,false,false -2024-01-11,000002.SZ,false,false,false,true,true,false,false -2024-01-11,000003.SZ,false,false,false,true,true,false,false -2024-01-11,600001.SH,false,false,true,false,false,false,false -2024-01-12,000001.SZ,false,false,false,true,true,false,false -2024-01-12,000002.SZ,false,false,false,true,true,false,false -2024-01-12,000003.SZ,false,false,false,true,true,false,false -2024-01-12,600001.SH,false,false,false,true,true,false,false diff --git a/data/demo/factors.csv b/data/demo/factors.csv deleted file mode 100644 index 537868d..0000000 --- a/data/demo/factors.csv +++ /dev/null @@ -1,37 +0,0 @@ -date,symbol,market_cap_bn,free_float_cap_bn,pe_ttm -2024-01-02,000001.SZ,38,24,18 -2024-01-02,000002.SZ,45,30,20 -2024-01-02,000003.SZ,65,40,15 -2024-01-02,600001.SH,85,55,13 -2024-01-03,000001.SZ,39,24.5,18 -2024-01-03,000002.SZ,46,30.5,20 -2024-01-03,000003.SZ,64,39.5,15 -2024-01-03,600001.SH,85,55,13 -2024-01-04,000001.SZ,40,25,18 -2024-01-04,000002.SZ,47,31,20 -2024-01-04,000003.SZ,63,39,15 -2024-01-04,600001.SH,86,55.5,13 -2024-01-05,000001.SZ,41,25.5,18 -2024-01-05,000002.SZ,48,32,20 -2024-01-05,000003.SZ,62,38.5,15 -2024-01-05,600001.SH,86,56,13 -2024-01-08,000001.SZ,42,26,18 -2024-01-08,000002.SZ,50,33,21 -2024-01-08,000003.SZ,61,38,15 -2024-01-08,600001.SH,87,56.5,13 -2024-01-09,000001.SZ,44,27,19 -2024-01-09,000002.SZ,52,34,21 -2024-01-09,000003.SZ,60,37.5,15 -2024-01-09,600001.SH,88,57,13 -2024-01-10,000001.SZ,43,26.5,19 -2024-01-10,000002.SZ,53,34.5,21 -2024-01-10,000003.SZ,59,37,15 -2024-01-10,600001.SH,89,57.5,13 -2024-01-11,000001.SZ,42,26,18 -2024-01-11,000002.SZ,52,34,21 -2024-01-11,000003.SZ,58,36.5,15 -2024-01-11,600001.SH,90,58,13 -2024-01-12,000001.SZ,40,25,18 -2024-01-12,000002.SZ,50,33,20 -2024-01-12,000003.SZ,57,36,15 -2024-01-12,600001.SH,92,59,13 diff --git a/data/demo/instruments.csv b/data/demo/instruments.csv deleted file mode 100644 index f73bb3b..0000000 --- a/data/demo/instruments.csv +++ /dev/null @@ -1,5 +0,0 @@ -symbol,name,board -000001.SZ,Alpha Components,Main -000002.SZ,Beta Precision,Main -000003.SZ,Charlie Materials,Main -600001.SH,Delta Industrials,Main diff --git a/data/demo/market.csv b/data/demo/market.csv deleted file mode 100644 index f10bfd5..0000000 --- a/data/demo/market.csv +++ /dev/null @@ -1,37 +0,0 @@ -date,symbol,open,high,low,close,prev_close,volume,paused -2024-01-02,000001.SZ,10.0,10.2,9.9,10.1,9.8,1200000,false -2024-01-02,000002.SZ,11.0,11.3,10.9,11.2,10.8,1100000,false -2024-01-02,000003.SZ,8.0,8.1,7.8,7.9,8.0,900000,false -2024-01-02,600001.SH,15.0,15.2,14.9,15.1,15.0,800000,false -2024-01-03,000001.SZ,10.2,10.5,10.1,10.4,10.1,1250000,false -2024-01-03,000002.SZ,11.2,11.6,11.1,11.5,11.2,1120000,false -2024-01-03,000003.SZ,7.8,7.9,7.3,7.4,7.9,930000,false -2024-01-03,600001.SH,15.1,15.3,15.0,15.2,15.1,820000,false -2024-01-04,000001.SZ,10.5,10.8,10.4,10.7,10.4,1280000,false -2024-01-04,000002.SZ,11.4,11.9,11.3,11.8,11.5,1150000,false -2024-01-04,000003.SZ,7.3,7.4,7.0,7.1,7.4,940000,false -2024-01-04,600001.SH,15.2,15.5,15.1,15.4,15.2,830000,false -2024-01-05,000001.SZ,10.8,11.1,10.7,11.0,10.7,1300000,false -2024-01-05,000002.SZ,11.9,12.1,11.8,12.0,11.8,1180000,false -2024-01-05,000003.SZ,7.0,7.1,6.8,6.9,7.1,950000,false -2024-01-05,600001.SH,15.4,15.6,15.3,15.5,15.4,840000,false -2024-01-08,000001.SZ,11.1,11.6,11.0,11.5,11.0,1400000,false -2024-01-08,000002.SZ,12.1,12.5,12.0,12.4,12.0,1200000,false -2024-01-08,000003.SZ,7.0,7.3,6.9,7.2,6.9,980000,false -2024-01-08,600001.SH,15.5,15.7,15.4,15.6,15.5,850000,false -2024-01-09,000001.SZ,11.6,12.4,11.5,12.3,11.5,1500000,false -2024-01-09,000002.SZ,12.5,12.9,12.4,12.8,12.4,1250000,false -2024-01-09,000003.SZ,7.2,7.5,7.1,7.4,7.2,990000,false -2024-01-09,600001.SH,15.6,15.7,15.4,15.5,15.6,860000,false -2024-01-10,000001.SZ,12.2,12.3,11.9,12.0,12.3,1450000,false -2024-01-10,000002.SZ,12.7,12.8,12.5,12.6,12.8,1220000,false -2024-01-10,000003.SZ,7.5,7.6,7.4,7.5,7.4,1000000,false -2024-01-10,600001.SH,15.4,15.5,15.1,15.2,15.5,870000,false -2024-01-11,000001.SZ,12.0,12.1,11.5,11.6,12.0,1420000,false -2024-01-11,000002.SZ,12.5,12.6,12.1,12.2,12.6,1210000,false -2024-01-11,000003.SZ,7.4,7.5,7.2,7.3,7.5,980000,false -2024-01-11,600001.SH,15.2,15.2,15.2,15.2,15.2,0,true -2024-01-12,000001.SZ,11.5,11.6,11.1,11.2,11.6,1380000,false -2024-01-12,000002.SZ,12.1,12.2,11.8,11.9,12.2,1190000,false -2024-01-12,000003.SZ,7.2,7.2,6.9,7.0,7.3,960000,false -2024-01-12,600001.SH,14.8,15.0,14.7,14.9,15.2,850000,false diff --git a/scripts/verify-no-legacy-data-source.sh b/scripts/verify-no-legacy-data-source.sh index 2e7107c..d4beaba 100755 --- a/scripts/verify-no-legacy-data-source.sh +++ b/scripts/verify-no-legacy-data-source.sh @@ -117,4 +117,22 @@ if [[ -n "$truth_csv_hits" ]]; then fail "CSV truth stock-list overrides are not allowed in fidc-backtest-engine runtime; use Source Lake runtime spec selection only" "$truth_csv_hits" fi +csv_snapshot_loader_hits="$( + rg -n \ + --glob '!**/.git/**' \ + --glob '!**/target/**' \ + --glob '!**/docs/**' \ + --glob '!**/*.md' \ + --glob '!**/tests/**' \ + --glob '!**/*test*.rs' \ + --glob '!scripts/verify-no-legacy-data-source.sh' \ + 'from_csv_dir|from_partitioned_dir|instruments\.csv|candidate_flags\.csv|market\.csv|benchmark\.csv' \ + crates Cargo.toml \ + 2>/dev/null || true +)" + +if [[ -n "$csv_snapshot_loader_hits" ]]; then + fail "CSV snapshot loaders are not allowed in fidc-backtest-engine runtime; construct DataSet from Source Lake components" "$csv_snapshot_loader_hits" +fi + printf '[OK] fidc-backtest-engine has no legacy runtime data-source references\n'