zsb/fidc-backtest-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

清理CSV回测demo入口

Browse Source
This commit is contained in:
boris
2026-07-03 23:40:33 +08:00
parent fea09ce93c
commit 73dd006bb2
18 changed files with 593 additions and 1711 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
-10
View File
@@ -37,16 +37,6 @@ version = "2.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
[[package]]
name = "bt-demo"
version = "0.1.0"
dependencies = [
"chrono",
"fidc-core",
"serde",
"serde_json",
]
[[package]] [[package]]
name = "bumpalo" name = "bumpalo"
version = "3.20.2" version = "3.20.2"
Cargo.toml
-1
View File
@@ -1,7 +1,6 @@
[workspace] [workspace]
members = [ members = [
"crates/fidc-core", "crates/fidc-core",
"crates/bt-demo",
] ]
resolver = "2" resolver = "2"
README.md
+2 -40
View File
@@ -22,7 +22,6 @@
. .
├── Cargo.toml ├── Cargo.toml
├── crates ├── crates
│ ├── bt-demo
│ └── fidc-core │ └── fidc-core
│ └── src │ └── src
│ ├── broker.rs │ ├── broker.rs
@@ -37,7 +36,6 @@
│ ├── scheduler.rs │ ├── scheduler.rs
│ ├── strategy.rs │ ├── strategy.rs
│ └── strategy_ai.rs │ └── strategy_ai.rs
├── data/demo
└── docs └── docs
``` ```
@@ -96,45 +94,9 @@
## 运行方式 ## 运行方式
默认运行仓库 demo 数据： `fidc-backtest-engine` 不再维护本地 CSV demo、partitioned snapshot 目录或导出融合表作为运行入口。生产和集成回测由 `fidc-backtest-service` runner 创建 `DataSet`，数据来自 Strategy Factory Source Lake 的 Arrow/Parquet、manifest/data_epoch 缓存和运行时逻辑视图。
```bash 本仓库只保留核心库构建和测试入口：
cargo run --bin bt-demo
```
运行平台内置微盘策略：
```bash
FIDC_BT_STRATEGY=omni-microcap \
FIDC_BT_SIGNAL_SYMBOL=000001.SH \
cargo run --release --bin bt-demo
```
接入真实分区 snapshot 目录：
```bash
FIDC_BT_DATA_LAYOUT=partitioned \
FIDC_BT_DATA_DIR=/path/to/snapshots \
FIDC_BT_SIGNAL_SYMBOL=000001.SH \
cargo run --bin bt-demo
```
约定目录结构：
```text
snapshots/
├── instruments.csv
├── benchmark/YYYY/MM/*.csv
├── market/YYYY/MM/*.csv
├── factors/YYYY/MM/*.csv
└── candidates/YYYY/MM/*.csv
```
运行后默认生成：
- `output/demo/equity_curve.csv`
- `output/demo/trades.csv`
- `output/demo/holdings_summary.csv`
## 测试与构建 ## 测试与构建
crates/bt-demo/Cargo.toml
-12
View File
@@ -1,12 +0,0 @@
[package]
name = "bt-demo"
version.workspace = true
edition.workspace = true
license.workspace = true
authors.workspace = true
[dependencies]
chrono = { workspace = true }
fidc-core = { path = "../fidc-core" }
serde = { workspace = true }
serde_json = "1"
crates/bt-demo/src/main.rs
-567
View File
@@ -1,567 +0,0 @@
use std::collections::BTreeSet;
use std::error::Error;
use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};
use chrono::{NaiveDate, NaiveTime};
use fidc_core::{
BacktestConfig, BacktestEngine, BenchmarkSnapshot, BrokerSimulator, ChinaAShareCostModel,
ChinaEquityRuleHooks, CnSmallCapRotationConfig, CnSmallCapRotationStrategy, DailyEquityPoint,
DataSet, FillEvent, HoldingSummary, OmniMicroCapConfig, OmniMicroCapStrategy, PortfolioState,
PriceField, Strategy, StrategyContext,
};
use serde_json::json;
fn main() -> Result<(), Box<dyn Error>> {
let root = workspace_root();
let data_dir = std::env::var("FIDC_BT_DATA_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| root.join("data/demo"));
let data_layout = std::env::var("FIDC_BT_DATA_LAYOUT").unwrap_or_else(|_| "flat".to_string());
let output_dir = std::env::var("FIDC_BT_OUTPUT_DIR")
.map(PathBuf::from)
.unwrap_or_else(|_| root.join("output/demo"));
let json_output = std::env::var("FIDC_BT_JSON")
.map(|value| value == "1" || value.eq_ignore_ascii_case("true"))
.unwrap_or(false);
fs::create_dir_all(&output_dir)?;
let data = if data_layout == "partitioned" {
DataSet::from_partitioned_dir(&data_dir)?
} else {
DataSet::from_csv_dir(&data_dir)?
};
let strategy_name =
std::env::var("FIDC_BT_STRATEGY").unwrap_or_else(|_| "cn-smallcap-rotation".to_string());
let debug_date = std::env::var("FIDC_BT_DEBUG_DATE")
.ok()
.filter(|value| !value.trim().is_empty())
.map(|value| NaiveDate::parse_from_str(value.trim(), "%Y-%m-%d"))
.transpose()?;
let decision_lag = std::env::var("FIDC_BT_DECISION_LAG")
.ok()
.and_then(|value| value.parse::<usize>().ok());
let execution_price =
std::env::var("FIDC_BT_EXECUTION_PRICE")
.ok()
.map(|value| match value.as_str() {
"close" => PriceField::Close,
"last" => PriceField::Last,
_ => PriceField::Open,
});
let initial_cash = std::env::var("FIDC_BT_INITIAL_CASH")
.ok()
.and_then(|value| value.parse::<f64>().ok());
let start_date = std::env::var("FIDC_BT_START_DATE")
.ok()
.filter(|value| !value.trim().is_empty())
.map(|value| NaiveDate::parse_from_str(value.trim(), "%Y-%m-%d"))
.transpose()?;
let end_date = std::env::var("FIDC_BT_END_DATE")
.ok()
.filter(|value| !value.trim().is_empty())
.map(|value| NaiveDate::parse_from_str(value.trim(), "%Y-%m-%d"))
.transpose()?;
let mut config = BacktestConfig {
initial_cash: initial_cash.unwrap_or(1_000_000.0),
benchmark_code: data.benchmark_code().to_string(),
start_date,
end_date,
decision_lag_trading_days: 1,
execution_price_field: PriceField::Open,
};
let result = match strategy_name.as_str() {
"cn-smallcap-rotation" | "cn-dyn-smallcap-band" => {
let mut strategy_cfg = if strategy_name == "cn-dyn-smallcap-band" {
CnSmallCapRotationConfig::cn_dyn_smallcap_band()
} else {
CnSmallCapRotationConfig::demo()
};
if strategy_cfg.strategy_name == "cn-smallcap-rotation" {
strategy_cfg.base_index_level = 3000.0;
strategy_cfg.base_cap_floor = 38.0;
strategy_cfg.cap_span = 25.0;
}
if let Ok(signal_symbol) = std::env::var("FIDC_BT_SIGNAL_SYMBOL") {
if !signal_symbol.trim().is_empty() {
strategy_cfg.signal_symbol = Some(signal_symbol);
}
}
config.decision_lag_trading_days = decision_lag.unwrap_or(1);
config.execution_price_field = execution_price.unwrap_or(PriceField::Open);
let strategy = CnSmallCapRotationStrategy::new(strategy_cfg);
let broker = BrokerSimulator::new_with_execution_price(
ChinaAShareCostModel::default(),
ChinaEquityRuleHooks::default(),
config.execution_price_field,
);
let mut engine = BacktestEngine::new(data, strategy, broker, config);
engine.run()?
}
"aiquant-v104" => {
let mut strategy_cfg = OmniMicroCapConfig::aiquant_v104();
if let Ok(signal_symbol) = std::env::var("FIDC_BT_SIGNAL_SYMBOL") {
if !signal_symbol.trim().is_empty() {
strategy_cfg.benchmark_signal_symbol = signal_symbol;
}
}
if let Some(date) = debug_date {
let eligible = data.eligible_universe_on(date);
eprintln!(
"DEBUG eligible_universe_on {} count={}",
date,
eligible.len()
);
for row in eligible.iter().take(20) {
eprintln!(" {} {:.6}", row.symbol, row.market_cap_bn);
}
let mut debug_strategy = OmniMicroCapStrategy::new(strategy_cfg.clone());
let debug_subscriptions = BTreeSet::new();
let decision = debug_strategy.on_day(&StrategyContext {
execution_date: date,
decision_date: date,
decision_index: 1,
data: &data,
portfolio: &PortfolioState::new(20_000.0),
futures_account: None,
open_orders: &[],
dynamic_universe: None,
subscriptions: &debug_subscriptions,
process_events: &[],
active_process_event: None,
active_datetime: None,
order_events: &[],
fills: &[],
})?;
eprintln!("DEBUG notes={:?}", decision.notes);
eprintln!("DEBUG diagnostics={:?}", decision.diagnostics);
return Ok(());
}
config.decision_lag_trading_days = decision_lag.unwrap_or(1);
config.execution_price_field = execution_price.unwrap_or(PriceField::Close);
config.initial_cash = initial_cash.unwrap_or(20_000.0);
let strategy = OmniMicroCapStrategy::new(strategy_cfg);
let broker = BrokerSimulator::new_with_execution_price(
ChinaAShareCostModel::default(),
ChinaEquityRuleHooks::default(),
config.execution_price_field,
);
let mut engine = BacktestEngine::new(data, strategy, broker, config);
engine.run()?
}
_ => {
let mut strategy_cfg = OmniMicroCapConfig::omni_microcap();
if let Ok(signal_symbol) = std::env::var("FIDC_BT_SIGNAL_SYMBOL") {
if !signal_symbol.trim().is_empty() {
strategy_cfg.benchmark_signal_symbol = signal_symbol;
}
}
if let Some(date) = debug_date {
let eligible = data.eligible_universe_on(date);
eprintln!(
"DEBUG eligible_universe_on {} count={}",
date,
eligible.len()
);
for row in eligible.iter().take(20) {
eprintln!(" {} {:.6}", row.symbol, row.market_cap_bn);
}
let mut debug_strategy = OmniMicroCapStrategy::new(strategy_cfg.clone());
let debug_subscriptions = BTreeSet::new();
let decision = debug_strategy.on_day(&StrategyContext {
execution_date: date,
decision_date: date,
decision_index: 1,
data: &data,
portfolio: &PortfolioState::new(10_000_000.0),
futures_account: None,
open_orders: &[],
dynamic_universe: None,
subscriptions: &debug_subscriptions,
process_events: &[],
active_process_event: None,
active_datetime: None,
order_events: &[],
fills: &[],
})?;
eprintln!("DEBUG notes={:?}", decision.notes);
eprintln!("DEBUG diagnostics={:?}", decision.diagnostics);
return Ok(());
}
config.decision_lag_trading_days = decision_lag.unwrap_or(0);
config.execution_price_field = execution_price.unwrap_or(PriceField::Last);
config.initial_cash = initial_cash.unwrap_or(10_000_000.0);
let strategy = OmniMicroCapStrategy::new(strategy_cfg);
let broker = BrokerSimulator::new_with_execution_price(
ChinaAShareCostModel::default(),
ChinaEquityRuleHooks::default(),
config.execution_price_field,
)
.with_intraday_execution_start_time(
NaiveTime::parse_from_str("10:18:00", "%H:%M:%S").expect("valid 10:18:00"),
)
.with_volume_limit(false)
.with_inactive_limit(false)
.with_liquidity_limit(false);
let mut engine = BacktestEngine::new(data, strategy, broker, config);
engine.run()?
}
};
write_equity_curve_csv(&output_dir.join("equity_curve.csv"), &result.equity_curve)?;
write_trades_csv(&output_dir.join("trades.csv"), &result.fills)?;
write_holdings_csv(
&output_dir.join("holdings_summary.csv"),
&result.holdings_summary,
)?;
let summary = build_summary(
&result.strategy_name,
&result.equity_curve,
&result.fills,
&result.holdings_summary,
result.benchmark_series.last(),
&output_dir,
);
print_summary(&summary, &result.equity_curve, &result.holdings_summary);
println!("Artifacts written under {}", output_dir.display());
if json_output {
println!("{}", serde_json::to_string(&summary)?);
}
Ok(())
}
fn workspace_root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../..")
.canonicalize()
.expect("workspace root")
}
fn write_equity_curve_csv(path: &Path, rows: &[DailyEquityPoint]) -> Result<(), Box<dyn Error>> {
let mut file = fs::File::create(path)?;
writeln!(
file,
"date,cash,market_value,total_equity,benchmark_close,benchmark_prev_close,notes,diagnostics"
)?;
for row in rows {
writeln!(
file,
"{},{:.2},{:.2},{:.2},{:.2},{:.2},{},{}",
row.date,
row.cash,
row.market_value,
row.total_equity,
row.benchmark_close,
row.benchmark_prev_close,
sanitize_csv_field(&row.notes),
sanitize_csv_field(&row.diagnostics),
)?;
}
Ok(())
}
fn write_trades_csv(path: &Path, rows: &[FillEvent]) -> Result<(), Box<dyn Error>> {
let mut file = fs::File::create(path)?;
writeln!(
file,
"date,symbol,side,quantity,price,gross_amount,commission,stamp_tax,net_cash_flow,reason"
)?;
for row in rows {
writeln!(
file,
"{},{},{:?},{},{:.2},{:.2},{:.2},{:.2},{:.2},{}",
row.date,
row.symbol,
row.side,
row.quantity,
row.price,
row.gross_amount,
row.commission,
row.stamp_tax,
row.net_cash_flow,
sanitize_csv_field(&row.reason),
)?;
}
Ok(())
}
fn write_holdings_csv(path: &Path, rows: &[HoldingSummary]) -> Result<(), Box<dyn Error>> {
let mut file = fs::File::create(path)?;
writeln!(
file,
"date,symbol,quantity,average_cost,last_price,market_value,unrealized_pnl,realized_pnl"
)?;
for row in rows {
writeln!(
file,
"{},{},{},{:.2},{:.2},{:.2},{:.2},{:.2}",
row.date,
row.symbol,
row.quantity,
row.average_cost,
row.last_price,
row.market_value,
row.unrealized_pnl,
row.realized_pnl,
)?;
}
Ok(())
}
fn sanitize_csv_field(text: &str) -> String {
text.replace(',', ";")
}
#[derive(Debug, serde::Serialize)]
struct RunSummary {
strategy: String,
start_date: String,
end_date: String,
start_equity: f64,
final_equity: f64,
total_return: f64,
trade_count: usize,
holding_count: usize,
benchmark_code: Option<String>,
benchmark_last_close: Option<f64>,
output_dir: String,
diagnostics: serde_json::Value,
warnings: Vec<String>,
equity_preview: Vec<serde_json::Value>,
trades_preview: Vec<serde_json::Value>,
}
fn build_summary(
strategy_name: &str,
equity_curve: &[DailyEquityPoint],
fills: &[FillEvent],
holdings: &[HoldingSummary],
benchmark_last: Option<&BenchmarkSnapshot>,
output_dir: &Path,
) -> RunSummary {
let first = equity_curve.first();
let last = equity_curve.last();
let start_equity = first.map(|row| row.total_equity).unwrap_or_default();
let final_equity = last.map(|row| row.total_equity).unwrap_or_default();
let total_return = if start_equity.abs() < f64::EPSILON {
0.0
} else {
(final_equity / start_equity) - 1.0
};
let diagnostics = extract_diagnostics(equity_curve);
let warnings = build_warnings(fills, holdings, &diagnostics);
let equity_preview = equity_curve
.iter()
.rev()
.take(5)
.collect::<Vec<_>>()
.into_iter()
.rev()
.map(|row| {
json!({
"date": row.date.to_string(),
"cash": row.cash,
"marketValue": row.market_value,
"totalEquity": row.total_equity,
"benchmarkClose": row.benchmark_close,
"benchmarkPrevClose": row.benchmark_prev_close,
"notes": row.notes,
"diagnostics": row.diagnostics,
})
})
.collect::<Vec<_>>();
let trades_preview = fills
.iter()
.rev()
.take(10)
.collect::<Vec<_>>()
.into_iter()
.rev()
.map(|row| {
json!({
"date": row.date.to_string(),
"symbol": row.symbol,
"side": format!("{:?}", row.side),
"quantity": row.quantity,
"price": row.price,
"grossAmount": row.gross_amount,
"netCashFlow": row.net_cash_flow,
"reason": row.reason,
})
})
.collect::<Vec<_>>();
RunSummary {
strategy: strategy_name.to_string(),
start_date: first.map(|row| row.date.to_string()).unwrap_or_default(),
end_date: last.map(|row| row.date.to_string()).unwrap_or_default(),
start_equity,
final_equity,
total_return,
trade_count: fills.len(),
holding_count: holdings.len(),
benchmark_code: benchmark_last.map(|row| row.benchmark.clone()),
benchmark_last_close: benchmark_last.map(|row| row.close),
output_dir: output_dir.display().to_string(),
diagnostics,
warnings,
equity_preview,
trades_preview,
}
}
fn extract_diagnostics(equity_curve: &[DailyEquityPoint]) -> serde_json::Value {
let last = equity_curve.last();
let text = last.map(|row| row.diagnostics.as_str()).unwrap_or("");
let notes = last.map(|row| row.notes.as_str()).unwrap_or("");
let mut map = serde_json::Map::new();
map.insert("latestText".to_string(), json!(text));
map.insert("latestNotes".to_string(), json!(notes));
map.insert("equityPointCount".to_string(), json!(equity_curve.len()));
for part in text.split(" | ") {
let part = part.trim();
if let Some(rest) = part.strip_prefix("selection_diag ") {
for token in rest.split_whitespace() {
if let Some((k, v)) = token.split_once('=') {
map.insert(k.to_string(), parse_diag_value(v));
}
}
} else if let Some(rest) = part.strip_prefix("selection_band ") {
for token in rest.split_whitespace() {
if let Some((k, v)) = token.split_once('=') {
map.insert(k.to_string(), parse_diag_value(v));
}
}
} else if let Some(rest) =
part.strip_prefix("market_cap_missing likely blocks selection; sample=")
{
map.insert(
"marketCapMissingSample".to_string(),
json!(
rest.split('|')
.filter(|s| !s.is_empty())
.collect::<Vec<_>>()
),
);
} else if let Some(rest) = part.strip_prefix("selection_rejections sample=") {
map.insert(
"selectionRejectionsSample".to_string(),
json!(
rest.split(" | ")
.filter(|s| !s.is_empty())
.collect::<Vec<_>>()
),
);
} else if let Some(rest) = part.strip_prefix("ma_filter_rejections sample=") {
map.insert(
"maFilterRejectionsSample".to_string(),
json!(
rest.split('|')
.filter(|s| !s.is_empty())
.collect::<Vec<_>>()
),
);
} else if let Some(rest) = part.strip_prefix("selected=") {
map.insert("selectedLine".to_string(), json!(rest));
}
}
serde_json::Value::Object(map)
}
fn parse_diag_value(value: &str) -> serde_json::Value {
if let Ok(v) = value.parse::<i64>() {
return json!(v);
}
if let Ok(v) = value.parse::<f64>() {
return json!(v);
}
json!(value)
}
fn build_warnings(
fills: &[FillEvent],
holdings: &[HoldingSummary],
diagnostics: &serde_json::Value,
) -> Vec<String> {
let mut warnings = Vec::new();
if fills.is_empty() {
warnings.push("本次回测没有产生任何成交。".to_string());
}
if holdings.is_empty() {
warnings.push("期末没有持仓。".to_string());
}
let selected_after_ma_is_empty = diagnostics
.get("selected_after_ma")
.and_then(|v| v.as_i64())
.unwrap_or(0)
== 0;
if selected_after_ma_is_empty && fills.is_empty() && holdings.is_empty() {
warnings
.push("最终没有股票通过完整选股链路，结果为空时请优先查看 diagnostics。".to_string());
}
if diagnostics
.get("market_cap_missing_count")
.and_then(|v| v.as_i64())
.unwrap_or(0)
> 0
{
warnings.push("存在 market_cap 缺失或非正值，当前会直接阻断该股票进入候选池。".to_string());
}
warnings
}
fn print_summary(
summary: &RunSummary,
equity_curve: &[DailyEquityPoint],
holdings: &[HoldingSummary],
) {
if equity_curve.is_empty() {
println!("No equity curve points generated.");
return;
}
println!("Strategy: {}", summary.strategy);
println!("Start equity: {:.2}", summary.start_equity);
println!("Final equity: {:.2}", summary.final_equity);
println!("Total return: {:.2}%", summary.total_return * 100.0);
println!("Trades: {}", summary.trade_count);
println!("Final holdings: {}", summary.holding_count);
if let (Some(code), Some(close)) = (&summary.benchmark_code, summary.benchmark_last_close) {
println!("Benchmark last close: {} {:.2}", code, close);
}
println!("Recent equity points:");
for point in equity_curve
.iter()
.rev()
.take(3)
.collect::<Vec<_>>()
.into_iter()
.rev()
{
println!(
" {} equity {:.2} cash {:.2} mv {:.2}",
point.date, point.total_equity, point.cash, point.market_value
);
}
if holdings.is_empty() {
println!("No holdings at the end of the demo run.");
} else {
println!("Ending holdings:");
for holding in holdings {
println!(
" {} qty {} mv {:.2} pnl {:.2}",
holding.symbol, holding.quantity, holding.market_value, holding.unrealized_pnl
);
}
}
}
crates/fidc-core/src/data.rs
+1 -677
View File
@@ -1,6 +1,4 @@
use std::collections::{BTreeMap, HashMap, HashSet}; use std::collections::{BTreeMap, HashMap, HashSet};
use std::fs;
use std::path::Path;
use std::sync::{Arc, OnceLock, RwLock}; use std::sync::{Arc, OnceLock, RwLock};
use chrono::{NaiveDate, NaiveDateTime}; use chrono::{NaiveDate, NaiveDateTime};
@@ -8,7 +6,7 @@ use serde::{Deserialize, Serialize};
use thiserror::Error; use thiserror::Error;
use crate::calendar::TradingCalendar; use crate::calendar::TradingCalendar;
use crate::futures::{FuturesCommissionType, FuturesTradingParameter}; use crate::futures::FuturesTradingParameter;
use crate::instrument::Instrument; use crate::instrument::Instrument;
use crate::risk_control::{ChinaAShareRiskControl, FidcRiskControlConfig}; use crate::risk_control::{ChinaAShareRiskControl, FidcRiskControlConfig};
@@ -58,18 +56,6 @@ mod datetime_format {
#[derive(Debug, Error)] #[derive(Debug, Error)]
pub enum DataSetError { pub enum DataSetError {
#[error("failed to read file {path}: {source}")]
Io {
path: String,
#[source]
source: std::io::Error,
},
#[error("invalid csv row in {path} at line {line}: {message}")]
InvalidRow {
path: String,
line: usize,
message: String,
},
#[error("benchmark file contains multiple benchmark codes")] #[error("benchmark file contains multiple benchmark codes")]
MultipleBenchmarks, MultipleBenchmarks,
#[error("missing data for {kind} on {date} / {symbol}")] #[error("missing data for {kind} on {date} / {symbol}")]
@@ -1006,96 +992,6 @@ pub struct DataSet {
} }
impl DataSet { impl DataSet {
pub fn from_csv_dir(path: &Path) -> Result<Self, DataSetError> {
let instruments = read_instruments(&path.join("instruments.csv"))?;
let market = read_market(&path.join("market.csv"))?;
let factors = read_factors(&path.join("factors.csv"))?;
let factor_texts = read_factor_texts(&path.join("factors.csv"))?;
let candidates = read_candidates(&path.join("candidate_flags.csv"))?;
let benchmarks = read_benchmarks(&path.join("benchmark.csv"))?;
let corporate_actions_path = path.join("corporate_actions.csv");
let corporate_actions = if corporate_actions_path.exists() {
read_corporate_actions(&corporate_actions_path)?
} else {
Vec::new()
};
let execution_quotes_path = path.join("execution_quotes.csv");
let execution_quotes = if execution_quotes_path.exists() {
read_execution_quotes(&execution_quotes_path)?
} else {
Vec::new()
};
let futures_params_path = path.join("futures_trading_parameters.csv");
let futures_params = if futures_params_path.exists() {
read_futures_trading_parameters(&futures_params_path)?
} else {
Vec::new()
};
let order_book_depth_path = path.join("order_book_depth.csv");
let order_book_depth = if order_book_depth_path.exists() {
read_order_book_depth(&order_book_depth_path)?
} else {
Vec::new()
};
Self::from_components_with_actions_quotes_futures_depth_and_factor_texts(
instruments,
market,
factors,
candidates,
benchmarks,
corporate_actions,
execution_quotes,
futures_params,
order_book_depth,
factor_texts,
)
}
pub fn from_partitioned_dir(path: &Path) -> Result<Self, DataSetError> {
let instruments = read_instruments(&path.join("instruments.csv"))?;
let benchmarks = read_partitioned_dir(&path.join("benchmark"), read_benchmarks)?;
let market = read_partitioned_dir(&path.join("market"), read_market)?;
let factors = read_partitioned_dir(&path.join("factors"), read_factors)?;
let factor_texts = read_partitioned_dir(&path.join("factors"), read_factor_texts)?;
let candidates = read_partitioned_dir(&path.join("candidates"), read_candidates)?;
let corporate_actions_dir = path.join("corporate_actions");
let corporate_actions = if corporate_actions_dir.exists() {
read_partitioned_dir(&corporate_actions_dir, read_corporate_actions)?
} else {
Vec::new()
};
let execution_quotes_dir = path.join("execution_quotes");
let execution_quotes = if execution_quotes_dir.exists() {
read_partitioned_dir(&execution_quotes_dir, read_execution_quotes)?
} else {
Vec::new()
};
let futures_params_dir = path.join("futures_trading_parameters");
let futures_params = if futures_params_dir.exists() {
read_partitioned_dir(&futures_params_dir, read_futures_trading_parameters)?
} else {
Vec::new()
};
let order_book_depth_dir = path.join("order_book_depth");
let order_book_depth = if order_book_depth_dir.exists() {
read_partitioned_dir(&order_book_depth_dir, read_order_book_depth)?
} else {
Vec::new()
};
Self::from_components_with_actions_quotes_futures_depth_and_factor_texts(
instruments,
market,
factors,
candidates,
benchmarks,
corporate_actions,
execution_quotes,
futures_params,
order_book_depth,
factor_texts,
)
}
pub fn from_components( pub fn from_components(
instruments: Vec<Instrument>, instruments: Vec<Instrument>,
market: Vec<DailyMarketSnapshot>, market: Vec<DailyMarketSnapshot>,
@@ -2665,157 +2561,6 @@ impl DataSet {
} }
} }
fn read_instruments(path: &Path) -> Result<Vec<Instrument>, DataSetError> {
let rows = read_rows(path)?;
let mut instruments = Vec::new();
for row in rows {
instruments.push(Instrument {
symbol: row.get(0)?.to_string(),
name: row.get(1)?.to_string(),
board: row.get(2)?.to_string(),
round_lot: row.parse_optional_u32(3).unwrap_or(100),
listed_at: row.parse_optional_date(4)?,
delisted_at: row.parse_optional_date(5)?,
status: row
.fields
.get(6)
.map(|value| value.trim())
.filter(|value| !value.is_empty())
.unwrap_or("active")
.to_string(),
});
}
Ok(instruments)
}
fn read_market(path: &Path) -> Result<Vec<DailyMarketSnapshot>, DataSetError> {
let rows = read_rows(path)?;
let mut snapshots = Vec::new();
for row in rows {
let open = row.parse_f64(2)?;
let close = row.parse_f64(5)?;
let prev_close = row.parse_f64(6)?;
let price_tick = row.parse_optional_f64(15).unwrap_or(0.01);
let derived_upper_limit = round_price_to_tick(prev_close * 1.10, price_tick);
let derived_lower_limit = round_price_to_tick(prev_close * 0.90, price_tick);
snapshots.push(DailyMarketSnapshot {
date: row.parse_date(0)?,
symbol: row.get(1)?.to_string(),
timestamp: row
.fields
.get(16)
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty()),
day_open: row.parse_optional_f64(11).unwrap_or(open),
open,
high: row.parse_f64(3)?,
low: row.parse_f64(4)?,
close,
last_price: row.parse_optional_f64(12).unwrap_or(close),
bid1: row.parse_optional_f64(13).unwrap_or(close),
ask1: row.parse_optional_f64(14).unwrap_or(close),
prev_close,
volume: row.parse_u64(7)?,
minute_volume: row.parse_optional_u64(17).unwrap_or_default(),
bid1_volume: row.parse_optional_u64(18).unwrap_or_default(),
ask1_volume: row.parse_optional_u64(19).unwrap_or_default(),
trading_phase: row
.fields
.get(20)
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty()),
paused: row.parse_bool(8)?,
upper_limit: row.parse_optional_f64(9).unwrap_or(derived_upper_limit),
lower_limit: row.parse_optional_f64(10).unwrap_or(derived_lower_limit),
price_tick,
});
}
Ok(snapshots)
}
fn read_factors(path: &Path) -> Result<Vec<DailyFactorSnapshot>, DataSetError> {
let rows = read_rows(path)?;
let mut snapshots = Vec::new();
for row in rows {
let (extra_factors, _) = parse_extra_factor_maps(&row);
snapshots.push(DailyFactorSnapshot {
date: row.parse_date(0)?,
symbol: row.get(1)?.to_string(),
market_cap_bn: row.parse_f64(2)?,
free_float_cap_bn: row.parse_f64(3)?,
pe_ttm: row.parse_f64(4)?,
turnover_ratio: row.parse_optional_f64(5),
effective_turnover_ratio: row.parse_optional_f64(6),
extra_factors,
});
}
Ok(snapshots)
}
fn read_factor_texts(path: &Path) -> Result<Vec<FactorTextValue>, DataSetError> {
let rows = read_rows(path)?;
let mut text_values = Vec::new();
for row in rows {
let date = row.parse_date(0)?;
let symbol = row.get(1)?.to_string();
let (_, extra_text_factors) = parse_extra_factor_maps(&row);
for (field, value) in extra_text_factors {
text_values.push(FactorTextValue {
date,
symbol: symbol.clone(),
field,
value,
});
}
}
Ok(text_values)
}
fn parse_extra_factor_maps(row: &CsvRow) -> (BTreeMap<String, f64>, BTreeMap<String, String>) {
let mut numeric = BTreeMap::new();
let mut text = BTreeMap::new();
for value in row.fields.get(7).into_iter().chain(row.fields.get(8)) {
merge_extra_factor_json(value, &mut numeric, &mut text);
}
(numeric, text)
}
fn merge_extra_factor_json(
raw: &str,
numeric: &mut BTreeMap<String, f64>,
text: &mut BTreeMap<String, String>,
) {
let trimmed = raw.trim();
if trimmed.is_empty() {
return;
}
let Ok(serde_json::Value::Object(map)) = serde_json::from_str::<serde_json::Value>(trimmed)
else {
return;
};
for (key, value) in map {
let key = normalize_field(&key);
if key.is_empty() {
continue;
}
match value {
serde_json::Value::Number(number) => {
if let Some(value) = number.as_f64().filter(|value| value.is_finite()) {
numeric.insert(key, value);
}
}
serde_json::Value::String(value) => {
text.insert(key, value);
}
serde_json::Value::Bool(value) => {
numeric.insert(key.clone(), if value { 1.0 } else { 0.0 });
text.insert(key, value.to_string());
}
_ => {}
}
}
}
fn normalized_aliases(values: &[String]) -> Vec<String> { fn normalized_aliases(values: &[String]) -> Vec<String> {
let mut aliases = Vec::new(); let mut aliases = Vec::new();
for value in values { for value in values {
@@ -3123,374 +2868,6 @@ fn take_last<T>(mut rows: Vec<T>, count: usize) -> Vec<T> {
rows.split_off(rows.len() - count) rows.split_off(rows.len() - count)
} }
fn read_candidates(path: &Path) -> Result<Vec<CandidateEligibility>, DataSetError> {
let rows = read_rows(path)?;
let mut snapshots = Vec::new();
for row in rows {
snapshots.push(CandidateEligibility {
date: row.parse_date(0)?,
symbol: row.get(1)?.to_string(),
is_st: row.parse_bool(2)?,
is_star_st: row.parse_bool(3)?,
is_new_listing: row.parse_bool(4)?,
is_paused: row.parse_bool(5)?,
allow_buy: row.parse_bool(6)?,
allow_sell: row.parse_bool(7)?,
is_kcb: row.parse_optional_bool(8).unwrap_or(false),
is_one_yuan: row.parse_optional_bool(9).unwrap_or(false),
risk_level_code: row
.fields
.get(10)
.map(String::as_str)
.map(str::trim)
.filter(|value| !value.is_empty())
.map(ToOwned::to_owned),
});
}
Ok(snapshots)
}
fn read_benchmarks(path: &Path) -> Result<Vec<BenchmarkSnapshot>, DataSetError> {
let rows = read_rows(path)?;
let mut snapshots = Vec::new();
for row in rows {
snapshots.push(BenchmarkSnapshot {
date: row.parse_date(0)?,
benchmark: row.get(1)?.to_string(),
open: row.parse_f64(2)?,
close: row.parse_f64(3)?,
prev_close: row.parse_f64(4)?,
volume: row.parse_u64(5)?,
});
}
Ok(snapshots)
}
fn read_corporate_actions(path: &Path) -> Result<Vec<CorporateAction>, DataSetError> {
let rows = read_rows(path)?;
let mut snapshots = Vec::new();
for row in rows {
let has_payable_date = row.fields.len() >= 10;
let payable_date = if has_payable_date {
row.parse_optional_date(2)?
} else {
None
};
let offset = if has_payable_date { 1 } else { 0 };
snapshots.push(CorporateAction {
date: row.parse_date(0)?,
symbol: row.get(1)?.to_string(),
payable_date,
share_cash: row.parse_optional_f64(2 + offset).unwrap_or(0.0),
share_bonus: row.parse_optional_f64(3 + offset).unwrap_or(0.0),
share_gift: row.parse_optional_f64(4 + offset).unwrap_or(0.0),
issue_quantity: row.parse_optional_f64(5 + offset).unwrap_or(0.0),
issue_price: row.parse_optional_f64(6 + offset).unwrap_or(0.0),
reform: row.parse_optional_bool(7 + offset).unwrap_or(false),
adjust_factor: row.parse_optional_f64(8 + offset),
successor_symbol: row
.fields
.get(9 + offset)
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty()),
successor_ratio: row.parse_optional_f64(10 + offset),
successor_cash: row.parse_optional_f64(11 + offset),
});
}
Ok(snapshots)
}
fn read_execution_quotes(path: &Path) -> Result<Vec<IntradayExecutionQuote>, DataSetError> {
let rows = read_rows(path)?;
let mut quotes = Vec::new();
for row in rows {
quotes.push(IntradayExecutionQuote {
date: row.parse_date(0)?,
symbol: row.get(1)?.to_string(),
timestamp: row.parse_datetime(2)?,
last_price: row.parse_optional_f64(3).unwrap_or_default(),
bid1: row.parse_optional_f64(4).unwrap_or_default(),
ask1: row.parse_optional_f64(5).unwrap_or_default(),
bid1_volume: row.parse_optional_u64(6).unwrap_or_default(),
ask1_volume: row.parse_optional_u64(7).unwrap_or_default(),
volume_delta: row.parse_optional_u64(8).unwrap_or_default(),
amount_delta: row.parse_optional_f64(9).unwrap_or_default(),
trading_phase: row
.fields
.get(10)
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty()),
});
}
Ok(quotes)
}
fn read_order_book_depth(path: &Path) -> Result<Vec<IntradayOrderBookDepthLevel>, DataSetError> {
let rows = read_rows(path)?;
let mut levels = Vec::new();
for row in rows {
levels.push(IntradayOrderBookDepthLevel {
date: row.parse_date(0)?,
symbol: row.get(1)?.to_string(),
timestamp: row.parse_datetime(2)?,
level: row
.parse_optional_u32(3)
.unwrap_or(1)
.clamp(1, u8::MAX as u32) as u8,
bid_price: row.parse_optional_f64(4).unwrap_or_default(),
bid_volume: row.parse_optional_u64(5).unwrap_or_default(),
ask_price: row.parse_optional_f64(6).unwrap_or_default(),
ask_volume: row.parse_optional_u64(7).unwrap_or_default(),
});
}
Ok(levels)
}
fn read_futures_trading_parameters(
path: &Path,
) -> Result<Vec<FuturesTradingParameter>, DataSetError> {
let rows = read_rows(path)?;
let mut params = Vec::new();
for row in rows {
let first = row.get(0)?.trim();
let (effective_date, symbol_index) = if NaiveDate::parse_from_str(first, "%Y-%m-%d").is_ok()
{
(row.parse_optional_date(0)?, 1)
} else {
(None, 0)
};
params.push(FuturesTradingParameter {
effective_date,
symbol: row.get(symbol_index)?.to_string(),
contract_multiplier: row.parse_optional_f64(symbol_index + 1).unwrap_or(1.0),
long_margin_rate: row.parse_optional_f64(symbol_index + 2).unwrap_or(0.0),
short_margin_rate: row.parse_optional_f64(symbol_index + 3).unwrap_or(0.0),
commission_type: row
.fields
.get(symbol_index + 4)
.map(|value| FuturesCommissionType::parse(value))
.unwrap_or(FuturesCommissionType::ByMoney),
open_commission_ratio: row.parse_optional_f64(symbol_index + 5).unwrap_or(0.0),
close_commission_ratio: row.parse_optional_f64(symbol_index + 6).unwrap_or(0.0),
close_today_commission_ratio: row
.parse_optional_f64(symbol_index + 7)
.unwrap_or_else(|| row.parse_optional_f64(symbol_index + 6).unwrap_or(0.0)),
price_tick: row.parse_optional_f64(symbol_index + 8).unwrap_or(1.0),
});
}
Ok(params)
}
struct CsvRow {
path: String,
line: usize,
fields: Vec<String>,
}
impl CsvRow {
fn get(&self, index: usize) -> Result<&str, DataSetError> {
self.fields
.get(index)
.map(String::as_str)
.ok_or_else(|| DataSetError::InvalidRow {
path: self.path.clone(),
line: self.line,
message: format!("missing column {index}"),
})
}
fn parse_date(&self, index: usize) -> Result<NaiveDate, DataSetError> {
NaiveDate::parse_from_str(self.get(index)?, "%Y-%m-%d").map_err(|err| {
DataSetError::InvalidRow {
path: self.path.clone(),
line: self.line,
message: format!("invalid date: {err}"),
}
})
}
fn parse_f64(&self, index: usize) -> Result<f64, DataSetError> {
self.get(index)?
.parse::<f64>()
.map_err(|err| DataSetError::InvalidRow {
path: self.path.clone(),
line: self.line,
message: format!("invalid f64: {err}"),
})
}
fn parse_u64(&self, index: usize) -> Result<u64, DataSetError> {
self.get(index)?
.parse::<u64>()
.map_err(|err| DataSetError::InvalidRow {
path: self.path.clone(),
line: self.line,
message: format!("invalid u64: {err}"),
})
}
fn parse_optional_f64(&self, index: usize) -> Option<f64> {
self.fields.get(index).and_then(|value| {
let trimmed = value.trim();
if trimmed.is_empty() {
None
} else {
trimmed.parse::<f64>().ok()
}
})
}
fn parse_bool(&self, index: usize) -> Result<bool, DataSetError> {
self.get(index)?
.parse::<bool>()
.map_err(|err| DataSetError::InvalidRow {
path: self.path.clone(),
line: self.line,
message: format!("invalid bool: {err}"),
})
}
fn parse_optional_bool(&self, index: usize) -> Option<bool> {
self.fields
.get(index)
.and_then(|value| value.parse::<bool>().ok())
}
fn parse_optional_date(&self, index: usize) -> Result<Option<NaiveDate>, DataSetError> {
let Some(value) = self.fields.get(index) else {
return Ok(None);
};
let trimmed = value.trim();
if trimmed.is_empty() {
return Ok(None);
}
NaiveDate::parse_from_str(trimmed, "%Y-%m-%d")
.map(Some)
.map_err(|err| DataSetError::InvalidRow {
path: self.path.clone(),
line: self.line,
message: format!("invalid optional date: {err}"),
})
}
fn parse_datetime(&self, index: usize) -> Result<NaiveDateTime, DataSetError> {
NaiveDateTime::parse_from_str(self.get(index)?, "%Y-%m-%d %H:%M:%S").map_err(|err| {
DataSetError::InvalidRow {
path: self.path.clone(),
line: self.line,
message: format!("invalid datetime: {err}"),
}
})
}
fn parse_optional_u32(&self, index: usize) -> Option<u32> {
self.fields.get(index).and_then(|value| {
let trimmed = value.trim();
if trimmed.is_empty() {
None
} else {
trimmed.parse::<u32>().ok()
}
})
}
fn parse_optional_u64(&self, index: usize) -> Option<u64> {
self.fields.get(index).and_then(|value| {
let trimmed = value.trim();
if trimmed.is_empty() {
None
} else {
trimmed.parse::<u64>().ok()
}
})
}
}
fn read_partitioned_dir<T, F>(dir: &Path, mut loader: F) -> Result<Vec<T>, DataSetError>
where
F: FnMut(&Path) -> Result<Vec<T>, DataSetError>,
{
let mut rows = Vec::new();
let mut stack = vec![dir.to_path_buf()];
while let Some(current_dir) = stack.pop() {
let mut entries = fs::read_dir(&current_dir)
.map_err(|source| DataSetError::Io {
path: current_dir.display().to_string(),
source,
})?
.collect::<Result<Vec<_>, _>>()
.map_err(|source| DataSetError::Io {
path: current_dir.display().to_string(),
source,
})?;
entries.sort_by_key(|entry| entry.path());
for entry in entries.into_iter().rev() {
let path = entry.path();
if path.is_dir() {
stack.push(path);
continue;
}
if path.extension().and_then(|x| x.to_str()) != Some("csv") {
continue;
}
rows.extend(loader(&path)?);
}
}
Ok(rows)
}
fn read_rows(path: &Path) -> Result<Vec<CsvRow>, DataSetError> {
let content = fs::read_to_string(path).map_err(|source| DataSetError::Io {
path: path.display().to_string(),
source,
})?;
let mut rows = Vec::new();
for (line_idx, line) in content.lines().enumerate() {
let line_no = line_idx + 1;
if line_no == 1 || line.trim().is_empty() {
continue;
}
rows.push(CsvRow {
path: path.display().to_string(),
line: line_no,
fields: split_csv_line(line),
});
}
Ok(rows)
}
fn split_csv_line(line: &str) -> Vec<String> {
let mut fields = Vec::new();
let mut field = String::new();
let mut chars = line.trim_start_matches('\u{feff}').chars().peekable();
let mut in_quotes = false;
while let Some(ch) = chars.next() {
match ch {
'"' if in_quotes && chars.peek() == Some(&'"') => {
field.push('"');
chars.next();
}
'"' => {
in_quotes = !in_quotes;
}
',' if !in_quotes => {
fields.push(field.trim().to_string());
field.clear();
}
_ => field.push(ch),
}
}
fields.push(field.trim().to_string());
fields
}
fn group_by_date<T, F>(rows: Vec<T>, mut date_of: F) -> BTreeMap<NaiveDate, Vec<T>> fn group_by_date<T, F>(rows: Vec<T>, mut date_of: F) -> BTreeMap<NaiveDate, Vec<T>>
where where
F: FnMut(&T) -> NaiveDate, F: FnMut(&T) -> NaiveDate,
@@ -3549,15 +2926,6 @@ fn collect_benchmark_code(benchmarks: &[BenchmarkSnapshot]) -> Result<String, Da
} }
} }
fn round_price_to_tick(value: f64, tick: f64) -> f64 {
let effective_tick = if tick.is_finite() && tick > 0.0 {
tick
} else {
0.01
};
((value / effective_tick).round() * effective_tick * 10000.0).round() / 10000.0
}
fn prefix_sums(values: &[f64]) -> Vec<f64> { fn prefix_sums(values: &[f64]) -> Vec<f64> {
let mut prefix = Vec::with_capacity(values.len() + 1); let mut prefix = Vec::with_capacity(values.len() + 1);
prefix.push(0.0); prefix.push(0.0);
@@ -3770,15 +3138,6 @@ fn instrument_passes_baseline_selection(instrument: Option<&Instrument>, date: N
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use std::time::{SystemTime, UNIX_EPOCH};
fn temp_csv_path(name: &str) -> std::path::PathBuf {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_nanos();
std::env::temp_dir().join(format!("{}_{}_{}.csv", name, std::process::id(), nanos))
}
fn market_row(date: &str, prev_close: f64, volume: u64) -> DailyMarketSnapshot { fn market_row(date: &str, prev_close: f64, volume: u64) -> DailyMarketSnapshot {
DailyMarketSnapshot { DailyMarketSnapshot {
@@ -4193,39 +3552,4 @@ mod tests {
Some((200.0 + 9_999.0) / 2.0) Some((200.0 + 9_999.0) / 2.0)
); );
} }
#[test]
fn reads_mixed_numeric_and_text_extra_factors_from_quoted_csv_json() {
let path = temp_csv_path("mixed_factor_maps");
fs::write(
&path,
concat!(
"date,symbol,market_cap_bn,free_float_cap_bn,pe_ttm,turnover_ratio,effective_turnover_ratio,extra_factors\n",
"2025-01-02,000001.SZ,12,10,8,1,1,\"{\"\"custom_alpha\"\":7,\"\"industry_name\"\":\"\"electronics,hardware\"\",\"\"flag\"\":true}\"\n"
),
)
.unwrap();
let factors = read_factors(&path).unwrap();
let text_factors = read_factor_texts(&path).unwrap();
fs::remove_file(&path).ok();
assert_eq!(factors.len(), 1);
assert_eq!(
factors[0].extra_factors.get("custom_alpha").copied(),
Some(7.0)
);
assert_eq!(factors[0].extra_factors.get("flag").copied(), Some(1.0));
assert_eq!(text_factors.len(), 2);
assert!(
text_factors
.iter()
.any(|row| row.field == "industry_name" && row.value == "electronics,hardware")
);
assert!(
text_factors
.iter()
.any(|row| row.field == "flag" && row.value == "true")
);
}
} }
crates/fidc-core/tests/partitioned_loader.rs
-93
View File
@@ -1,93 +0,0 @@
use fidc_core::DataSet;
use std::fs;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
fn temp_dir() -> PathBuf {
let uniq = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("clock")
.as_nanos();
let dir = std::env::temp_dir().join(format!("fidc-bt-partitioned-{uniq}"));
fs::create_dir_all(&dir).expect("mkdir temp");
dir
}
#[test]
fn can_load_partitioned_snapshot_dir() {
let dir = temp_dir();
fs::create_dir_all(dir.join("benchmark/2024/01")).unwrap();
fs::create_dir_all(dir.join("market/2024/01")).unwrap();
fs::create_dir_all(dir.join("factors/2024/01")).unwrap();
fs::create_dir_all(dir.join("candidates/2024/01")).unwrap();
fs::create_dir_all(dir.join("corporate_actions/2024/01")).unwrap();
fs::write(
dir.join("instruments.csv"),
"symbol,name,board,round_lot,listed_at,delisted_at,status\n000001.SZ,PingAn,SZ,100,2020-01-01,,active\n",
)
.unwrap();
fs::write(
dir.join("benchmark/2024/01/2024-01-02.csv"),
"date,benchmark,open,close,prev_close,volume\n2024-01-02,CSI300.DEMO,2990,3000,2980,100000000\n",
)
.unwrap();
fs::write(
dir.join("market/2024/01/2024-01-02.csv"),
"date,symbol,open,high,low,close,prev_close,volume,paused,upper_limit,lower_limit,day_open,last_price,bid1,ask1,price_tick\n2024-01-02,000001.SZ,10,10.5,9.9,10.2,10,100000,false,11,9,10.1,10.15,10.14,10.16,0.01\n",
)
.unwrap();
fs::write(
dir.join("factors/2024/01/2024-01-02.csv"),
"date,symbol,market_cap_bn,free_float_cap_bn,pe_ttm,turnover_ratio,effective_turnover_ratio\n2024-01-02,000001.SZ,40,35,12,3.2,2.1\n",
)
.unwrap();
fs::write(
dir.join("candidates/2024/01/2024-01-02.csv"),
"date,symbol,is_st,is_new_listing,is_paused,allow_buy,allow_sell,is_kcb,is_one_yuan\n2024-01-02,000001.SZ,false,false,false,true,true,false,false\n",
)
.unwrap();
fs::write(
dir.join("corporate_actions/2024/01/2024-01-02.csv"),
"date,symbol,payable_date,share_cash,share_bonus,share_gift,issue_quantity,issue_price,reform,adjust_factor\n2024-01-02,000001.SZ,2024-01-05,0.5,0.1,0.0,0,0,false,1.05\n",
)
.unwrap();
let data = DataSet::from_partitioned_dir(&dir).expect("partitioned dataset");
assert_eq!(data.benchmark_code(), "CSI300.DEMO");
assert!(
data.market_snapshots_on(chrono::NaiveDate::from_ymd_opt(2024, 1, 2).unwrap())
.len()
== 1
);
let market_rows =
data.market_snapshots_on(chrono::NaiveDate::from_ymd_opt(2024, 1, 2).unwrap());
let snapshot = market_rows.first().expect("market snapshot");
assert_eq!(snapshot.day_open, 10.1);
assert_eq!(snapshot.last_price, 10.15);
assert_eq!(snapshot.price_tick, 0.01);
assert_eq!(
data.instruments()
.get("000001.SZ")
.expect("instrument")
.round_lot,
100
);
assert_eq!(
data.instruments()
.get("000001.SZ")
.expect("instrument")
.listed_at,
Some(chrono::NaiveDate::from_ymd_opt(2020, 1, 1).unwrap())
);
let actions = data.corporate_actions_on(chrono::NaiveDate::from_ymd_opt(2024, 1, 2).unwrap());
assert_eq!(actions.len(), 1);
assert_eq!(
actions[0].payable_date,
Some(chrono::NaiveDate::from_ymd_opt(2024, 1, 5).unwrap())
);
assert!((actions[0].share_cash - 0.5).abs() < 1e-9);
assert!((actions[0].split_ratio() - 1.1).abs() < 1e-9);
let _ = fs::remove_dir_all(&dir);
}
crates/fidc-core/tests/strategy_selection.rs
+572 -6
View File
@@ -1,15 +1,582 @@
use chrono::NaiveDate; use chrono::NaiveDate;
use fidc_core::{ use fidc_core::{
CnSmallCapRotationConfig, CnSmallCapRotationStrategy, DataSet, OmniMicroCapConfig, BenchmarkSnapshot, CandidateEligibility, CnSmallCapRotationConfig, CnSmallCapRotationStrategy,
DailyFactorSnapshot, DailyMarketSnapshot, DataSet, Instrument, OmniMicroCapConfig,
OmniMicroCapStrategy, PortfolioState, Strategy, StrategyContext, OmniMicroCapStrategy, PortfolioState, Strategy, StrategyContext,
}; };
use std::collections::BTreeSet; use std::collections::BTreeSet;
use std::path::PathBuf;
fn d(value: &str) -> NaiveDate {
NaiveDate::parse_from_str(value, "%Y-%m-%d").unwrap()
}
fn instrument(symbol: &str, name: &str) -> Instrument {
Instrument {
symbol: symbol.to_string(),
name: name.to_string(),
board: "Main".to_string(),
round_lot: 100,
listed_at: None,
delisted_at: None,
status: "active".to_string(),
}
}
fn market(
date: &str,
symbol: &str,
open: f64,
high: f64,
low: f64,
close: f64,
prev_close: f64,
volume: u64,
paused: bool,
) -> DailyMarketSnapshot {
DailyMarketSnapshot {
date: d(date),
symbol: symbol.to_string(),
timestamp: None,
day_open: open,
open,
high,
low,
close,
last_price: close,
bid1: close,
ask1: close,
prev_close,
volume,
minute_volume: 0,
bid1_volume: 0,
ask1_volume: 0,
trading_phase: None,
paused,
upper_limit: (prev_close * 1.10 * 100.0).round() / 100.0,
lower_limit: (prev_close * 0.90 * 100.0).round() / 100.0,
price_tick: 0.01,
}
}
fn factor(
date: &str,
symbol: &str,
market_cap_bn: f64,
free_float_cap_bn: f64,
) -> DailyFactorSnapshot {
DailyFactorSnapshot {
date: d(date),
symbol: symbol.to_string(),
market_cap_bn,
free_float_cap_bn,
pe_ttm: 18.0,
turnover_ratio: None,
effective_turnover_ratio: None,
extra_factors: Default::default(),
}
}
fn candidate(
date: &str,
symbol: &str,
is_new_listing: bool,
is_paused: bool,
allow_buy: bool,
allow_sell: bool,
) -> CandidateEligibility {
CandidateEligibility {
date: d(date),
symbol: symbol.to_string(),
is_st: false,
is_star_st: false,
is_new_listing,
is_paused,
allow_buy,
allow_sell,
is_kcb: false,
is_one_yuan: false,
risk_level_code: None,
}
}
fn benchmark(date: &str, open: f64, close: f64, prev_close: f64, volume: u64) -> BenchmarkSnapshot {
BenchmarkSnapshot {
date: d(date),
benchmark: "CSI300.DEMO".to_string(),
open,
close,
prev_close,
volume,
}
}
fn strategy_test_dataset() -> DataSet {
let dates = [
"2024-01-02",
"2024-01-03",
"2024-01-04",
"2024-01-05",
"2024-01-08",
"2024-01-09",
"2024-01-10",
"2024-01-11",
"2024-01-12",
];
let instruments = vec![
instrument("000001.SZ", "Alpha Components"),
instrument("000002.SZ", "Beta Precision"),
instrument("000003.SZ", "Charlie Materials"),
instrument("600001.SH", "Delta Industrials"),
];
let market = vec![
market(
"2024-01-02",
"000001.SZ",
10.0,
10.2,
9.9,
10.1,
9.8,
1_200_000,
false,
),
market(
"2024-01-02",
"000002.SZ",
11.0,
11.3,
10.9,
11.2,
10.8,
1_100_000,
false,
),
market(
"2024-01-02",
"000003.SZ",
8.0,
8.1,
7.8,
7.9,
8.0,
900_000,
false,
),
market(
"2024-01-02",
"600001.SH",
15.0,
15.2,
14.9,
15.1,
15.0,
800_000,
false,
),
market(
"2024-01-03",
"000001.SZ",
10.2,
10.5,
10.1,
10.4,
10.1,
1_250_000,
false,
),
market(
"2024-01-03",
"000002.SZ",
11.2,
11.6,
11.1,
11.5,
11.2,
1_120_000,
false,
),
market(
"2024-01-03",
"000003.SZ",
7.8,
7.9,
7.3,
7.4,
7.9,
930_000,
false,
),
market(
"2024-01-03",
"600001.SH",
15.1,
15.3,
15.0,
15.2,
15.1,
820_000,
false,
),
market(
"2024-01-04",
"000001.SZ",
10.5,
10.8,
10.4,
10.7,
10.4,
1_280_000,
false,
),
market(
"2024-01-04",
"000002.SZ",
11.4,
11.9,
11.3,
11.8,
11.5,
1_150_000,
false,
),
market(
"2024-01-04",
"000003.SZ",
7.3,
7.4,
7.0,
7.1,
7.4,
940_000,
false,
),
market(
"2024-01-04",
"600001.SH",
15.2,
15.5,
15.1,
15.4,
15.2,
830_000,
false,
),
market(
"2024-01-05",
"000001.SZ",
10.8,
11.1,
10.7,
11.0,
10.7,
1_300_000,
false,
),
market(
"2024-01-05",
"000002.SZ",
11.9,
12.1,
11.8,
12.0,
11.8,
1_180_000,
false,
),
market(
"2024-01-05",
"000003.SZ",
7.0,
7.1,
6.8,
6.9,
7.1,
950_000,
false,
),
market(
"2024-01-05",
"600001.SH",
15.4,
15.6,
15.3,
15.5,
15.4,
840_000,
false,
),
market(
"2024-01-08",
"000001.SZ",
11.1,
11.6,
11.0,
11.5,
11.0,
1_400_000,
false,
),
market(
"2024-01-08",
"000002.SZ",
12.1,
12.5,
12.0,
12.4,
12.0,
1_200_000,
false,
),
market(
"2024-01-08",
"000003.SZ",
7.0,
7.3,
6.9,
7.2,
6.9,
980_000,
false,
),
market(
"2024-01-08",
"600001.SH",
15.5,
15.7,
15.4,
15.6,
15.5,
850_000,
false,
),
market(
"2024-01-09",
"000001.SZ",
11.6,
12.4,
11.5,
12.3,
11.5,
1_500_000,
false,
),
market(
"2024-01-09",
"000002.SZ",
12.5,
12.9,
12.4,
12.8,
12.4,
1_250_000,
false,
),
market(
"2024-01-09",
"000003.SZ",
7.2,
7.5,
7.1,
7.4,
7.2,
990_000,
false,
),
market(
"2024-01-09",
"600001.SH",
15.6,
15.7,
15.4,
15.5,
15.6,
860_000,
false,
),
market(
"2024-01-10",
"000001.SZ",
12.2,
12.3,
11.9,
12.0,
12.3,
1_450_000,
false,
),
market(
"2024-01-10",
"000002.SZ",
12.7,
12.8,
12.5,
12.6,
12.8,
1_220_000,
false,
),
market(
"2024-01-10",
"000003.SZ",
7.5,
7.6,
7.4,
7.5,
7.4,
1_000_000,
false,
),
market(
"2024-01-10",
"600001.SH",
15.4,
15.5,
15.1,
15.2,
15.5,
870_000,
false,
),
market(
"2024-01-11",
"000001.SZ",
12.0,
12.1,
11.5,
11.6,
12.0,
1_420_000,
false,
),
market(
"2024-01-11",
"000002.SZ",
12.5,
12.6,
12.1,
12.2,
12.6,
1_210_000,
false,
),
market(
"2024-01-11",
"000003.SZ",
7.4,
7.5,
7.2,
7.3,
7.5,
980_000,
false,
),
market(
"2024-01-11",
"600001.SH",
15.2,
15.2,
15.2,
15.2,
15.2,
0,
true,
),
market(
"2024-01-12",
"000001.SZ",
11.5,
11.6,
11.1,
11.2,
11.6,
1_380_000,
false,
),
market(
"2024-01-12",
"000002.SZ",
12.1,
12.2,
11.8,
11.9,
12.2,
1_190_000,
false,
),
market(
"2024-01-12",
"000003.SZ",
7.2,
7.2,
6.9,
7.0,
7.3,
960_000,
false,
),
market(
"2024-01-12",
"600001.SH",
14.8,
15.0,
14.7,
14.9,
15.2,
850_000,
false,
),
];
let factors = dates
.iter()
.enumerate()
.flat_map(|(idx, date)| {
let i = idx as f64;
[
factor(date, "000001.SZ", 38.0 + i, 24.0 + i * 0.5),
factor(date, "000002.SZ", 45.0 + i, 30.0 + i * 0.5),
factor(date, "000003.SZ", 65.0 - i, 40.0 - i * 0.5),
factor(date, "600001.SH", 85.0 + i, 55.0 + i * 0.5),
]
})
.collect::<Vec<_>>();
let candidates = dates
.iter()
.flat_map(|date| {
let first_two = *date == "2024-01-02" || *date == "2024-01-03";
let paused_600001 = *date == "2024-01-11";
[
candidate(date, "000001.SZ", first_two, false, !first_two, true),
candidate(date, "000002.SZ", false, false, true, true),
candidate(date, "000003.SZ", false, false, true, true),
candidate(
date,
"600001.SH",
false,
paused_600001,
!paused_600001,
!paused_600001,
),
]
})
.collect::<Vec<_>>();
let benchmarks = vec![
benchmark("2024-01-02", 2990.0, 3000.0, 2980.0, 100_000_000),
benchmark("2024-01-03", 3005.0, 3020.0, 3000.0, 102_000_000),
benchmark("2024-01-04", 3025.0, 3050.0, 3020.0, 105_000_000),
benchmark("2024-01-05", 3055.0, 3080.0, 3050.0, 108_000_000),
benchmark("2024-01-08", 3085.0, 3110.0, 3080.0, 109_000_000),
benchmark("2024-01-09", 3100.0, 3090.0, 3110.0, 107_000_000),
benchmark("2024-01-10", 3080.0, 3040.0, 3090.0, 111_000_000),
benchmark("2024-01-11", 3030.0, 2990.0, 3040.0, 115_000_000),
benchmark("2024-01-12", 2980.0, 2950.0, 2990.0, 118_000_000),
];
DataSet::from_components(instruments, market, factors, candidates, benchmarks)
.expect("strategy test dataset")
}
#[test] #[test]
fn strategy_emits_target_weights_and_diagnostics() { fn strategy_emits_target_weights_and_diagnostics() {
let data_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../data/demo"); let data = strategy_test_dataset();
let data = DataSet::from_csv_dir(&data_dir).expect("demo data");
let decision_date = NaiveDate::from_ymd_opt(2024, 1, 10).unwrap(); let decision_date = NaiveDate::from_ymd_opt(2024, 1, 10).unwrap();
let execution_date = NaiveDate::from_ymd_opt(2024, 1, 11).unwrap(); let execution_date = NaiveDate::from_ymd_opt(2024, 1, 11).unwrap();
let portfolio = PortfolioState::new(1_000_000.0); let portfolio = PortfolioState::new(1_000_000.0);
@@ -53,8 +620,7 @@ fn strategy_emits_target_weights_and_diagnostics() {
#[test] #[test]
fn omni_strategy_emits_same_day_decision() { fn omni_strategy_emits_same_day_decision() {
let data_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../data/demo"); let data = strategy_test_dataset();
let data = DataSet::from_csv_dir(&data_dir).expect("demo data");
let execution_date = NaiveDate::from_ymd_opt(2024, 1, 10).unwrap(); let execution_date = NaiveDate::from_ymd_opt(2024, 1, 10).unwrap();
let portfolio = PortfolioState::new(1_000_000.0); let portfolio = PortfolioState::new(1_000_000.0);
let mut cfg = OmniMicroCapConfig::omni_microcap(); let mut cfg = OmniMicroCapConfig::omni_microcap();
data/demo/000852量价微盘平台策略.engine-script.txt
-63
View File
@@ -1,63 +0,0 @@
let refresh_rate = 15;
let stocknum = 40;
let close_rate = 1.07;
let loss_rate = 0.93;
let rsi_rate = 1.0001;
let trade_rate = 0.5;
let xs = 4 / 500;
let base_index_level = 2000;
let base_cap_floor = 3;
let base_cap_ceiling = 28;
fn band_start(current_price, base_index_level, xs, base_cap_floor) {
if current_price == base_index_level {
base_cap_floor
} else if current_price > 0 {
round((current_price - base_index_level) * xs + base_cap_floor)
} else {
base_cap_floor
}
}
fn band_end(current_price, base_index_level, xs, base_cap_ceiling) {
if current_price == base_index_level {
base_cap_ceiling
} else if current_price > 0 {
round((current_price - base_index_level) * xs + base_cap_ceiling)
} else {
base_cap_ceiling
}
}
strategy("microcap_volume_trend_000852") {
market("CN_A")
benchmark("000852.SH")
signal("000852.SH")
rebalance.every_days(refresh_rate).at("10:18")
universe.exclude("paused", "st", "kcb", "one_yuan", "new_listing")
selection.limit(stocknum)
selection.market_cap_band(
field="market_cap",
lower=band_start(signal_close, base_index_level, xs, base_cap_floor),
upper=band_end(signal_close, base_index_level, xs, base_cap_ceiling)
)
risk.index_exposure(
signal_ma5 > signal_ma10 * rsi_rate ? 1.0 : trade_rate
)
filter.stock_expr(
stock_ma5 > stock_ma10 * rsi_rate &&
stock_ma10 > stock_ma30 * rsi_rate &&
rolling_mean("volume", 5) < rolling_mean("volume", 60)
)
risk.take_profit(close_rate)
risk.stop_loss(loss_rate)
allocation.buy_scale(touched_upper_limit ? 1.0 : trade_rate)
ordering.rank_by("market_cap", "asc")
}
data/demo/000852量价微盘平台策略.strategy_spec.json
-41
View File
@@ -1,41 +0,0 @@
{
"strategyId": "microcap_volume_trend_000852",
"version": "2",
"parser": "omniquant-engine-script-v2",
"market": "CN_A",
"signalSymbol": "000852.SH",
"benchmark": {
"instrumentId": "000852.SH",
"fallbackInstrumentId": "000852.SH"
},
"engineConfig": {
"market": "CN_A",
"signalSymbol": "000852.SH",
"benchmarkSymbol": "000852.SH",
"refreshRate": 15,
"rankLimit": 40
},
"runtimeExpressions": {
"prelude": "let refresh_rate = 15;\nlet stocknum = 40;\nlet close_rate = 1.07;\nlet loss_rate = 0.93;\nlet rsi_rate = 1.0001;\nlet trade_rate = 0.5;\nlet xs = 4 / 500;\nlet base_index_level = 2000;\nlet base_cap_floor = 3;\nlet base_cap_ceiling = 28;\nfn band_start(current_price, base_index_level, xs, base_cap_floor) {\n if current_price == base_index_level {\n base_cap_floor\n } else if current_price > 0 {\n round((current_price - base_index_level) * xs + base_cap_floor)\n } else {\n base_cap_floor\n }\n}\nfn band_end(current_price, base_index_level, xs, base_cap_ceiling) {\n if current_price == base_index_level {\n base_cap_ceiling\n } else if current_price > 0 {\n round((current_price - base_index_level) * xs + base_cap_ceiling)\n } else {\n base_cap_ceiling\n }\n}",
"selection": {
"limitExpr": "stocknum",
"marketCapField": "market_cap",
"marketCapLowerExpr": "band_start(signal_close, base_index_level, xs, base_cap_floor)",
"marketCapUpperExpr": "band_end(signal_close, base_index_level, xs, base_cap_ceiling)",
"stockFilterExpr": "stock_ma5 > stock_ma10 * rsi_rate && stock_ma10 > stock_ma30 * rsi_rate && rolling_mean(\"volume\", 5) < rolling_mean(\"volume\", 60)"
},
"risk": {
"exposureExpr": "signal_ma5 > signal_ma10 * rsi_rate ? 1.0 : trade_rate",
"stopLossExpr": "loss_rate",
"takeProfitExpr": "close_rate"
},
"allocation": {
"buyScaleExpr": "touched_upper_limit ? 1.0 : trade_rate"
},
"ordering": {
"rankBy": "market_cap",
"rankExpr": "",
"rankOrder": "asc"
}
}
}
data/demo/ai_generated_000001_open_cap_band.engine-script.txt
-42
View File
@@ -1,42 +0,0 @@
let refresh_rate = 15;
let stocknum = 40;
let xs = 0.008;
let base_index_level = 2000;
let lower_offset = 3;
let upper_offset = 28;
fn cap_floor(current_price, base_index_level, xs, lower_offset) {
round((current_price - base_index_level) * xs + lower_offset)
}
fn cap_ceiling(current_price, base_index_level, xs, upper_offset) {
round((current_price - base_index_level) * xs + upper_offset)
}
strategy("ai_generated_000001_open_cap_band") {
market("CN_A")
benchmark("000852.SH")
signal("000001.SH")
rebalance.every_days(refresh_rate).at("10:18")
universe.exclude("paused", "st", "kcb", "one_yuan", "new_listing")
selection.limit(stocknum)
selection.market_cap_band(
field="market_cap",
lower=cap_floor(signal_open, base_index_level, xs, lower_offset),
upper=cap_ceiling(signal_open, base_index_level, xs, upper_offset)
)
filter.stock_expr(
stock_ma5 > stock_ma10 &&
stock_ma10 > stock_ma30 &&
rolling_mean("volume", 5) < rolling_mean("volume", 60) &&
!ends_with(symbol, ".BJ") &&
!at_upper_limit &&
!at_lower_limit
)
ordering.rank_by("market_cap", "asc")
}
data/demo/ai_generated_000001_open_cap_band.strategy_spec.json
-33
View File
@@ -1,33 +0,0 @@
{
"strategyId": "ai_generated_000001_open_cap_band",
"version": "2",
"parser": "omniquant-engine-script-v2",
"market": "CN_A",
"signalSymbol": "000001.SH",
"benchmark": {
"instrumentId": "000852.SH",
"fallbackInstrumentId": "000852.SH"
},
"engineConfig": {
"market": "CN_A",
"signalSymbol": "000001.SH",
"benchmarkSymbol": "000852.SH",
"refreshRate": 15,
"rankLimit": 40
},
"runtimeExpressions": {
"prelude": "let refresh_rate = 15;\nlet stocknum = 40;\nlet xs = 0.008;\nlet base_index_level = 2000;\nlet lower_offset = 3;\nlet upper_offset = 28;\n\nfn cap_floor(current_price, base_index_level, xs, lower_offset) {\nround((current_price - base_index_level) * xs + lower_offset)\n}\n\nfn cap_ceiling(current_price, base_index_level, xs, upper_offset) {\nround((current_price - base_index_level) * xs + upper_offset)\n}",
"selection": {
"limitExpr": "stocknum",
"marketCapField": "market_cap",
"marketCapLowerExpr": "cap_floor(signal_open, base_index_level, xs, lower_offset)",
"marketCapUpperExpr": "cap_ceiling(signal_open, base_index_level, xs, upper_offset)",
"stockFilterExpr": "stock_ma5 > stock_ma10 && stock_ma10 > stock_ma30 && rolling_mean(\"volume\", 5) < rolling_mean(\"volume\", 60) && !ends_with(symbol, \".BJ\") && !at_upper_limit && !at_lower_limit"
},
"ordering": {
"rankBy": "market_cap",
"rankExpr": "",
"rankOrder": "asc"
}
}
}
data/demo/benchmark.csv
-10
View File
@@ -1,10 +0,0 @@
date,benchmark,open,close,prev_close,volume
2024-01-02,CSI300.DEMO,2990,3000,2980,100000000
2024-01-03,CSI300.DEMO,3005,3020,3000,102000000
2024-01-04,CSI300.DEMO,3025,3050,3020,105000000
2024-01-05,CSI300.DEMO,3055,3080,3050,108000000
2024-01-08,CSI300.DEMO,3085,3110,3080,109000000
2024-01-09,CSI300.DEMO,3100,3090,3110,107000000
2024-01-10,CSI300.DEMO,3080,3040,3090,111000000
2024-01-11,CSI300.DEMO,3030,2990,3040,115000000
2024-01-12,CSI300.DEMO,2980,2950,2990,118000000
1 date benchmark open close prev_close volume
2 2024-01-02 CSI300.DEMO 2990 3000 2980 100000000
3 2024-01-03 CSI300.DEMO 3005 3020 3000 102000000
4 2024-01-04 CSI300.DEMO 3025 3050 3020 105000000
5 2024-01-05 CSI300.DEMO 3055 3080 3050 108000000
6 2024-01-08 CSI300.DEMO 3085 3110 3080 109000000
7 2024-01-09 CSI300.DEMO 3100 3090 3110 107000000
8 2024-01-10 CSI300.DEMO 3080 3040 3090 111000000
9 2024-01-11 CSI300.DEMO 3030 2990 3040 115000000
10 2024-01-12 CSI300.DEMO 2980 2950 2990 118000000
data/demo/candidate_flags.csv
-37
View File
@@ -1,37 +0,0 @@
date,symbol,is_st,is_new_listing,is_paused,allow_buy,allow_sell,is_kcb,is_one_yuan
2024-01-02,000001.SZ,false,true,false,false,true,false,false
2024-01-02,000002.SZ,false,false,false,true,true,false,false
2024-01-02,000003.SZ,false,false,false,true,true,false,false
2024-01-02,600001.SH,false,false,false,true,true,false,false
2024-01-03,000001.SZ,false,true,false,false,true,false,false
2024-01-03,000002.SZ,false,false,false,true,true,false,false
2024-01-03,000003.SZ,false,false,false,true,true,false,false
2024-01-03,600001.SH,false,false,false,true,true,false,false
2024-01-04,000001.SZ,false,false,false,true,true,false,false
2024-01-04,000002.SZ,false,false,false,true,true,false,false
2024-01-04,000003.SZ,false,false,false,true,true,false,false
2024-01-04,600001.SH,false,false,false,true,true,false,false
2024-01-05,000001.SZ,false,false,false,true,true,false,false
2024-01-05,000002.SZ,false,false,false,true,true,false,false
2024-01-05,000003.SZ,false,false,false,true,true,false,false
2024-01-05,600001.SH,false,false,false,true,true,false,false
2024-01-08,000001.SZ,false,false,false,true,true,false,false
2024-01-08,000002.SZ,false,false,false,true,true,false,false
2024-01-08,000003.SZ,false,false,false,true,true,false,false
2024-01-08,600001.SH,false,false,false,true,true,false,false
2024-01-09,000001.SZ,false,false,false,true,true,false,false
2024-01-09,000002.SZ,false,false,false,true,true,false,false
2024-01-09,000003.SZ,false,false,false,true,true,false,false
2024-01-09,600001.SH,false,false,false,true,true,false,false
2024-01-10,000001.SZ,false,false,false,true,true,false,false
2024-01-10,000002.SZ,false,false,false,true,true,false,false
2024-01-10,000003.SZ,false,false,false,true,true,false,false
2024-01-10,600001.SH,false,false,false,true,true,false,false
2024-01-11,000001.SZ,false,false,false,true,true,false,false
2024-01-11,000002.SZ,false,false,false,true,true,false,false
2024-01-11,000003.SZ,false,false,false,true,true,false,false
2024-01-11,600001.SH,false,false,true,false,false,false,false
2024-01-12,000001.SZ,false,false,false,true,true,false,false
2024-01-12,000002.SZ,false,false,false,true,true,false,false
2024-01-12,000003.SZ,false,false,false,true,true,false,false
2024-01-12,600001.SH,false,false,false,true,true,false,false
1 date symbol is_st is_new_listing is_paused allow_buy allow_sell is_kcb is_one_yuan
2 2024-01-02 000001.SZ false true false false true false false
3 2024-01-02 000002.SZ false false false true true false false
4 2024-01-02 000003.SZ false false false true true false false
5 2024-01-02 600001.SH false false false true true false false
6 2024-01-03 000001.SZ false true false false true false false
7 2024-01-03 000002.SZ false false false true true false false
8 2024-01-03 000003.SZ false false false true true false false
9 2024-01-03 600001.SH false false false true true false false
10 2024-01-04 000001.SZ false false false true true false false
11 2024-01-04 000002.SZ false false false true true false false
12 2024-01-04 000003.SZ false false false true true false false
13 2024-01-04 600001.SH false false false true true false false
14 2024-01-05 000001.SZ false false false true true false false
15 2024-01-05 000002.SZ false false false true true false false
16 2024-01-05 000003.SZ false false false true true false false
17 2024-01-05 600001.SH false false false true true false false
18 2024-01-08 000001.SZ false false false true true false false
19 2024-01-08 000002.SZ false false false true true false false
20 2024-01-08 000003.SZ false false false true true false false
21 2024-01-08 600001.SH false false false true true false false
22 2024-01-09 000001.SZ false false false true true false false
23 2024-01-09 000002.SZ false false false true true false false
24 2024-01-09 000003.SZ false false false true true false false
25 2024-01-09 600001.SH false false false true true false false
26 2024-01-10 000001.SZ false false false true true false false
27 2024-01-10 000002.SZ false false false true true false false
28 2024-01-10 000003.SZ false false false true true false false
29 2024-01-10 600001.SH false false false true true false false
30 2024-01-11 000001.SZ false false false true true false false
31 2024-01-11 000002.SZ false false false true true false false
32 2024-01-11 000003.SZ false false false true true false false
33 2024-01-11 600001.SH false false true false false false false
34 2024-01-12 000001.SZ false false false true true false false
35 2024-01-12 000002.SZ false false false true true false false
36 2024-01-12 000003.SZ false false false true true false false
37 2024-01-12 600001.SH false false false true true false false
data/demo/factors.csv
-37
View File
@@ -1,37 +0,0 @@
date,symbol,market_cap_bn,free_float_cap_bn,pe_ttm
2024-01-02,000001.SZ,38,24,18
2024-01-02,000002.SZ,45,30,20
2024-01-02,000003.SZ,65,40,15
2024-01-02,600001.SH,85,55,13
2024-01-03,000001.SZ,39,24.5,18
2024-01-03,000002.SZ,46,30.5,20
2024-01-03,000003.SZ,64,39.5,15
2024-01-03,600001.SH,85,55,13
2024-01-04,000001.SZ,40,25,18
2024-01-04,000002.SZ,47,31,20
2024-01-04,000003.SZ,63,39,15
2024-01-04,600001.SH,86,55.5,13
2024-01-05,000001.SZ,41,25.5,18
2024-01-05,000002.SZ,48,32,20
2024-01-05,000003.SZ,62,38.5,15
2024-01-05,600001.SH,86,56,13
2024-01-08,000001.SZ,42,26,18
2024-01-08,000002.SZ,50,33,21
2024-01-08,000003.SZ,61,38,15
2024-01-08,600001.SH,87,56.5,13
2024-01-09,000001.SZ,44,27,19
2024-01-09,000002.SZ,52,34,21
2024-01-09,000003.SZ,60,37.5,15
2024-01-09,600001.SH,88,57,13
2024-01-10,000001.SZ,43,26.5,19
2024-01-10,000002.SZ,53,34.5,21
2024-01-10,000003.SZ,59,37,15
2024-01-10,600001.SH,89,57.5,13
2024-01-11,000001.SZ,42,26,18
2024-01-11,000002.SZ,52,34,21
2024-01-11,000003.SZ,58,36.5,15
2024-01-11,600001.SH,90,58,13
2024-01-12,000001.SZ,40,25,18
2024-01-12,000002.SZ,50,33,20
2024-01-12,000003.SZ,57,36,15
2024-01-12,600001.SH,92,59,13
1 date symbol market_cap_bn free_float_cap_bn pe_ttm
2 2024-01-02 000001.SZ 38 24 18
3 2024-01-02 000002.SZ 45 30 20
4 2024-01-02 000003.SZ 65 40 15
5 2024-01-02 600001.SH 85 55 13
6 2024-01-03 000001.SZ 39 24.5 18
7 2024-01-03 000002.SZ 46 30.5 20
8 2024-01-03 000003.SZ 64 39.5 15
9 2024-01-03 600001.SH 85 55 13
10 2024-01-04 000001.SZ 40 25 18
11 2024-01-04 000002.SZ 47 31 20
12 2024-01-04 000003.SZ 63 39 15
13 2024-01-04 600001.SH 86 55.5 13
14 2024-01-05 000001.SZ 41 25.5 18
15 2024-01-05 000002.SZ 48 32 20
16 2024-01-05 000003.SZ 62 38.5 15
17 2024-01-05 600001.SH 86 56 13
18 2024-01-08 000001.SZ 42 26 18
19 2024-01-08 000002.SZ 50 33 21
20 2024-01-08 000003.SZ 61 38 15
21 2024-01-08 600001.SH 87 56.5 13
22 2024-01-09 000001.SZ 44 27 19
23 2024-01-09 000002.SZ 52 34 21
24 2024-01-09 000003.SZ 60 37.5 15
25 2024-01-09 600001.SH 88 57 13
26 2024-01-10 000001.SZ 43 26.5 19
27 2024-01-10 000002.SZ 53 34.5 21
28 2024-01-10 000003.SZ 59 37 15
29 2024-01-10 600001.SH 89 57.5 13
30 2024-01-11 000001.SZ 42 26 18
31 2024-01-11 000002.SZ 52 34 21
32 2024-01-11 000003.SZ 58 36.5 15
33 2024-01-11 600001.SH 90 58 13
34 2024-01-12 000001.SZ 40 25 18
35 2024-01-12 000002.SZ 50 33 20
36 2024-01-12 000003.SZ 57 36 15
37 2024-01-12 600001.SH 92 59 13
data/demo/instruments.csv
-5
View File
@@ -1,5 +0,0 @@
symbol,name,board
000001.SZ,Alpha Components,Main
000002.SZ,Beta Precision,Main
000003.SZ,Charlie Materials,Main
600001.SH,Delta Industrials,Main
1 symbol name board
2 000001.SZ Alpha Components Main
3 000002.SZ Beta Precision Main
4 000003.SZ Charlie Materials Main
5 600001.SH Delta Industrials Main
data/demo/market.csv
-37
View File
@@ -1,37 +0,0 @@
date,symbol,open,high,low,close,prev_close,volume,paused
2024-01-02,000001.SZ,10.0,10.2,9.9,10.1,9.8,1200000,false
2024-01-02,000002.SZ,11.0,11.3,10.9,11.2,10.8,1100000,false
2024-01-02,000003.SZ,8.0,8.1,7.8,7.9,8.0,900000,false
2024-01-02,600001.SH,15.0,15.2,14.9,15.1,15.0,800000,false
2024-01-03,000001.SZ,10.2,10.5,10.1,10.4,10.1,1250000,false
2024-01-03,000002.SZ,11.2,11.6,11.1,11.5,11.2,1120000,false
2024-01-03,000003.SZ,7.8,7.9,7.3,7.4,7.9,930000,false
2024-01-03,600001.SH,15.1,15.3,15.0,15.2,15.1,820000,false
2024-01-04,000001.SZ,10.5,10.8,10.4,10.7,10.4,1280000,false
2024-01-04,000002.SZ,11.4,11.9,11.3,11.8,11.5,1150000,false
2024-01-04,000003.SZ,7.3,7.4,7.0,7.1,7.4,940000,false
2024-01-04,600001.SH,15.2,15.5,15.1,15.4,15.2,830000,false
2024-01-05,000001.SZ,10.8,11.1,10.7,11.0,10.7,1300000,false
2024-01-05,000002.SZ,11.9,12.1,11.8,12.0,11.8,1180000,false
2024-01-05,000003.SZ,7.0,7.1,6.8,6.9,7.1,950000,false
2024-01-05,600001.SH,15.4,15.6,15.3,15.5,15.4,840000,false
2024-01-08,000001.SZ,11.1,11.6,11.0,11.5,11.0,1400000,false
2024-01-08,000002.SZ,12.1,12.5,12.0,12.4,12.0,1200000,false
2024-01-08,000003.SZ,7.0,7.3,6.9,7.2,6.9,980000,false
2024-01-08,600001.SH,15.5,15.7,15.4,15.6,15.5,850000,false
2024-01-09,000001.SZ,11.6,12.4,11.5,12.3,11.5,1500000,false
2024-01-09,000002.SZ,12.5,12.9,12.4,12.8,12.4,1250000,false
2024-01-09,000003.SZ,7.2,7.5,7.1,7.4,7.2,990000,false
2024-01-09,600001.SH,15.6,15.7,15.4,15.5,15.6,860000,false
2024-01-10,000001.SZ,12.2,12.3,11.9,12.0,12.3,1450000,false
2024-01-10,000002.SZ,12.7,12.8,12.5,12.6,12.8,1220000,false
2024-01-10,000003.SZ,7.5,7.6,7.4,7.5,7.4,1000000,false
2024-01-10,600001.SH,15.4,15.5,15.1,15.2,15.5,870000,false
2024-01-11,000001.SZ,12.0,12.1,11.5,11.6,12.0,1420000,false
2024-01-11,000002.SZ,12.5,12.6,12.1,12.2,12.6,1210000,false
2024-01-11,000003.SZ,7.4,7.5,7.2,7.3,7.5,980000,false
2024-01-11,600001.SH,15.2,15.2,15.2,15.2,15.2,0,true
2024-01-12,000001.SZ,11.5,11.6,11.1,11.2,11.6,1380000,false
2024-01-12,000002.SZ,12.1,12.2,11.8,11.9,12.2,1190000,false
2024-01-12,000003.SZ,7.2,7.2,6.9,7.0,7.3,960000,false
2024-01-12,600001.SH,14.8,15.0,14.7,14.9,15.2,850000,false
1 date symbol open high low close prev_close volume paused
2 2024-01-02 000001.SZ 10.0 10.2 9.9 10.1 9.8 1200000 false
3 2024-01-02 000002.SZ 11.0 11.3 10.9 11.2 10.8 1100000 false
4 2024-01-02 000003.SZ 8.0 8.1 7.8 7.9 8.0 900000 false
5 2024-01-02 600001.SH 15.0 15.2 14.9 15.1 15.0 800000 false
6 2024-01-03 000001.SZ 10.2 10.5 10.1 10.4 10.1 1250000 false
7 2024-01-03 000002.SZ 11.2 11.6 11.1 11.5 11.2 1120000 false
8 2024-01-03 000003.SZ 7.8 7.9 7.3 7.4 7.9 930000 false
9 2024-01-03 600001.SH 15.1 15.3 15.0 15.2 15.1 820000 false
10 2024-01-04 000001.SZ 10.5 10.8 10.4 10.7 10.4 1280000 false
11 2024-01-04 000002.SZ 11.4 11.9 11.3 11.8 11.5 1150000 false
12 2024-01-04 000003.SZ 7.3 7.4 7.0 7.1 7.4 940000 false
13 2024-01-04 600001.SH 15.2 15.5 15.1 15.4 15.2 830000 false
14 2024-01-05 000001.SZ 10.8 11.1 10.7 11.0 10.7 1300000 false
15 2024-01-05 000002.SZ 11.9 12.1 11.8 12.0 11.8 1180000 false
16 2024-01-05 000003.SZ 7.0 7.1 6.8 6.9 7.1 950000 false
17 2024-01-05 600001.SH 15.4 15.6 15.3 15.5 15.4 840000 false
18 2024-01-08 000001.SZ 11.1 11.6 11.0 11.5 11.0 1400000 false
19 2024-01-08 000002.SZ 12.1 12.5 12.0 12.4 12.0 1200000 false
20 2024-01-08 000003.SZ 7.0 7.3 6.9 7.2 6.9 980000 false
21 2024-01-08 600001.SH 15.5 15.7 15.4 15.6 15.5 850000 false
22 2024-01-09 000001.SZ 11.6 12.4 11.5 12.3 11.5 1500000 false
23 2024-01-09 000002.SZ 12.5 12.9 12.4 12.8 12.4 1250000 false
24 2024-01-09 000003.SZ 7.2 7.5 7.1 7.4 7.2 990000 false
25 2024-01-09 600001.SH 15.6 15.7 15.4 15.5 15.6 860000 false
26 2024-01-10 000001.SZ 12.2 12.3 11.9 12.0 12.3 1450000 false
27 2024-01-10 000002.SZ 12.7 12.8 12.5 12.6 12.8 1220000 false
28 2024-01-10 000003.SZ 7.5 7.6 7.4 7.5 7.4 1000000 false
29 2024-01-10 600001.SH 15.4 15.5 15.1 15.2 15.5 870000 false
30 2024-01-11 000001.SZ 12.0 12.1 11.5 11.6 12.0 1420000 false
31 2024-01-11 000002.SZ 12.5 12.6 12.1 12.2 12.6 1210000 false
32 2024-01-11 000003.SZ 7.4 7.5 7.2 7.3 7.5 980000 false
33 2024-01-11 600001.SH 15.2 15.2 15.2 15.2 15.2 0 true
34 2024-01-12 000001.SZ 11.5 11.6 11.1 11.2 11.6 1380000 false
35 2024-01-12 000002.SZ 12.1 12.2 11.8 11.9 12.2 1190000 false
36 2024-01-12 000003.SZ 7.2 7.2 6.9 7.0 7.3 960000 false
37 2024-01-12 600001.SH 14.8 15.0 14.7 14.9 15.2 850000 false
scripts/verify-no-legacy-data-source.sh
+18
View File
@@ -117,4 +117,22 @@ if [[ -n "$truth_csv_hits" ]]; then
fail "CSV truth stock-list overrides are not allowed in fidc-backtest-engine runtime; use Source Lake runtime spec selection only" "$truth_csv_hits" fail "CSV truth stock-list overrides are not allowed in fidc-backtest-engine runtime; use Source Lake runtime spec selection only" "$truth_csv_hits"
fi fi
csv_snapshot_loader_hits="$(
rg -n \
--glob '!**/.git/**' \
--glob '!**/target/**' \
--glob '!**/docs/**' \
--glob '!**/*.md' \
--glob '!**/tests/**' \
--glob '!**/*test*.rs' \
--glob '!scripts/verify-no-legacy-data-source.sh' \
'from_csv_dir|from_partitioned_dir|instruments\.csv|candidate_flags\.csv|market\.csv|benchmark\.csv' \
crates Cargo.toml \
2>/dev/null || true
)"
if [[ -n "$csv_snapshot_loader_hits" ]]; then
fail "CSV snapshot loaders are not allowed in fidc-backtest-engine runtime; construct DataSet from Source Lake components" "$csv_snapshot_loader_hits"
fi
printf '[OK] fidc-backtest-engine has no legacy runtime data-source references\n' printf '[OK] fidc-backtest-engine has no legacy runtime data-source references\n'