zsb/fidc-backtest-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

移除回测稠密索引性能回归

Browse Source
This commit is contained in:
boris
2026-08-24 11:53:12 +08:00
parent 0686532be0
commit 1d7ac19886
2 changed files with 39 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/fidc-core/src/data.rs
+28 -159
View File
@@ -78,12 +78,6 @@ pub enum PriceField {
Last, Last,
} }
#[derive(Debug, Clone, Copy)]
pub struct MarketRollingCursor {
current_end: usize,
decision_end: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DailyMarketSnapshot { pub struct DailyMarketSnapshot {
#[serde(with = "date_format")] #[serde(with = "date_format")]
@@ -554,18 +548,14 @@ impl AdjustedCloseSeries {
} }
fn current_moving_average(&self, date: NaiveDate, lookback: usize) -> Option<f64> { fn current_moving_average(&self, date: NaiveDate, lookback: usize) -> Option<f64> {
if lookback == 0 {
return None;
}
let end = match self.dates.binary_search(&date) { let end = match self.dates.binary_search(&date) {
Ok(index) => index + 1, Ok(index) => index + 1,
Err(0) => return None, Err(0) => return None,
Err(index) => index, Err(index) => index,
}; };
self.current_moving_average_at_end(end, lookback)
}
fn current_moving_average_at_end(&self, end: usize, lookback: usize) -> Option<f64> {
if lookback == 0 || end > self.dates.len() {
return None;
}
if end < lookback { if end < lookback {
return None; return None;
} }
@@ -587,18 +577,14 @@ impl AdjustedCloseSeries {
} }
fn decision_moving_average(&self, date: NaiveDate, lookback: usize) -> Option<f64> { fn decision_moving_average(&self, date: NaiveDate, lookback: usize) -> Option<f64> {
if lookback == 0 {
return None;
}
let end = match self.dates.binary_search(&date) { let end = match self.dates.binary_search(&date) {
Ok(index) => index, Ok(index) => index,
Err(0) => return None, Err(0) => return None,
Err(index) => index, Err(index) => index,
}; };
self.decision_moving_average_at_end(end, lookback)
}
fn decision_moving_average_at_end(&self, end: usize, lookback: usize) -> Option<f64> {
if lookback == 0 || end > self.dates.len() {
return None;
}
if end < lookback { if end < lookback {
return None; return None;
} }
@@ -868,10 +854,6 @@ impl SymbolPriceSeries {
fn decision_volume_moving_average(&self, date: NaiveDate, lookback: usize) -> Option<f64> { fn decision_volume_moving_average(&self, date: NaiveDate, lookback: usize) -> Option<f64> {
let end = self.previous_completed_end_index(date)?; let end = self.previous_completed_end_index(date)?;
self.decision_volume_moving_average_at_end(end, lookback)
}
fn decision_volume_moving_average_at_end(&self, end: usize, lookback: usize) -> Option<f64> {
self.valid_volume_window(end, lookback).map(|(start, end)| { self.valid_volume_window(end, lookback).map(|(start, end)| {
normalize_rolling_factor( normalize_rolling_factor(
(self.valid_volume_sum_prefix[end] - self.valid_volume_sum_prefix[start]) (self.valid_volume_sum_prefix[end] - self.valid_volume_sum_prefix[start])
@@ -883,10 +865,6 @@ impl SymbolPriceSeries {
fn current_volume_moving_average(&self, date: NaiveDate, lookback: usize) -> Option<f64> { fn current_volume_moving_average(&self, date: NaiveDate, lookback: usize) -> Option<f64> {
let end = self.end_index(date)?; let end = self.end_index(date)?;
self.current_volume_moving_average_at_end(end, lookback)
}
fn current_volume_moving_average_at_end(&self, end: usize, lookback: usize) -> Option<f64> {
self.valid_volume_window(end, lookback).map(|(start, end)| { self.valid_volume_window(end, lookback).map(|(start, end)| {
normalize_rolling_factor( normalize_rolling_factor(
(self.valid_volume_sum_prefix[end] - self.valid_volume_sum_prefix[start]) (self.valid_volume_sum_prefix[end] - self.valid_volume_sum_prefix[start])
@@ -920,13 +898,6 @@ impl SymbolPriceSeries {
Some((start, end)) Some((start, end))
} }
fn rolling_cursor(&self, date: NaiveDate) -> Option<MarketRollingCursor> {
Some(MarketRollingCursor {
current_end: self.end_index(date)?,
decision_end: self.previous_completed_end_index(date).unwrap_or(0),
})
}
fn valid_volume_values(&self, end: usize, lookback: usize) -> Option<Vec<f64>> { fn valid_volume_values(&self, end: usize, lookback: usize) -> Option<Vec<f64>> {
let (start, end) = self.valid_volume_window(end, lookback)?; let (start, end) = self.valid_volume_window(end, lookback)?;
let values = self.volumes[start..end] let values = self.volumes[start..end]
@@ -1154,14 +1125,13 @@ pub struct DataSet {
instruments: HashMap<String, Instrument>, instruments: HashMap<String, Instrument>,
calendar: TradingCalendar, calendar: TradingCalendar,
market_by_date: BTreeMap<NaiveDate, Vec<Arc<DailyMarketSnapshot>>>, market_by_date: BTreeMap<NaiveDate, Vec<Arc<DailyMarketSnapshot>>>,
market_row_index_by_date: Arc<BTreeMap<NaiveDate, Vec<u32>>>, market_symbol_ids_by_date: Arc<BTreeMap<NaiveDate, Vec<u32>>>,
factor_by_date: BTreeMap<NaiveDate, Vec<Arc<DailyFactorSnapshot>>>, factor_by_date: BTreeMap<NaiveDate, Vec<Arc<DailyFactorSnapshot>>>,
factor_row_index_by_date: Arc<BTreeMap<NaiveDate, Vec<u32>>>,
factor_symbol_ids_by_date: Arc<BTreeMap<NaiveDate, Vec<u32>>>, factor_symbol_ids_by_date: Arc<BTreeMap<NaiveDate, Vec<u32>>>,
factor_text_by_date: BTreeMap<NaiveDate, Vec<FactorTextValue>>, factor_text_by_date: BTreeMap<NaiveDate, Vec<FactorTextValue>>,
factor_text_index: HashMap<(NaiveDate, String, String), FactorTextValue>, factor_text_index: HashMap<(NaiveDate, String, String), FactorTextValue>,
candidate_by_date: BTreeMap<NaiveDate, Vec<Arc<CandidateEligibility>>>, candidate_by_date: BTreeMap<NaiveDate, Vec<Arc<CandidateEligibility>>>,
candidate_row_index_by_date: Arc<BTreeMap<NaiveDate, Vec<u32>>>, candidate_symbol_ids_by_date: Arc<BTreeMap<NaiveDate, Vec<u32>>>,
corporate_actions_by_date: BTreeMap<NaiveDate, Vec<CorporateAction>>, corporate_actions_by_date: BTreeMap<NaiveDate, Vec<CorporateAction>>,
execution_quotes_by_date: HashMap<NaiveDate, HashMap<String, Vec<IntradayExecutionQuote>>>, execution_quotes_by_date: HashMap<NaiveDate, HashMap<String, Vec<IntradayExecutionQuote>>>,
order_book_depth_index: HashMap<(NaiveDate, String), Vec<IntradayOrderBookDepthLevel>>, order_book_depth_index: HashMap<(NaiveDate, String), Vec<IntradayOrderBookDepthLevel>>,
@@ -1381,20 +1351,16 @@ impl DataSet {
&factor_by_date, &factor_by_date,
&candidate_by_date, &candidate_by_date,
); );
let market_row_index_by_date = let market_symbol_ids_by_date =
build_group_row_indices(&market_by_date, &symbol_id_by_code, |item| { build_group_symbol_ids(&market_by_date, &symbol_id_by_code, |item| {
item.symbol.as_str()
});
let factor_row_index_by_date =
build_group_row_indices(&factor_by_date, &symbol_id_by_code, |item| {
item.symbol.as_str() item.symbol.as_str()
}); });
let factor_symbol_ids_by_date = let factor_symbol_ids_by_date =
build_group_symbol_ids(&factor_by_date, &symbol_id_by_code, |item| { build_group_symbol_ids(&factor_by_date, &symbol_id_by_code, |item| {
item.symbol.as_str() item.symbol.as_str()
}); });
let candidate_row_index_by_date = let candidate_symbol_ids_by_date =
build_group_row_indices(&candidate_by_date, &symbol_id_by_code, |item| { build_group_symbol_ids(&candidate_by_date, &symbol_id_by_code, |item| {
item.symbol.as_str() item.symbol.as_str()
}); });
let mut market_series_by_symbol_id = vec![None; symbol_id_by_code.len()]; let mut market_series_by_symbol_id = vec![None; symbol_id_by_code.len()];
@@ -1425,14 +1391,13 @@ impl DataSet {
instruments, instruments,
calendar, calendar,
market_by_date, market_by_date,
market_row_index_by_date: Arc::new(market_row_index_by_date), market_symbol_ids_by_date: Arc::new(market_symbol_ids_by_date),
factor_by_date, factor_by_date,
factor_row_index_by_date: Arc::new(factor_row_index_by_date),
factor_symbol_ids_by_date: Arc::new(factor_symbol_ids_by_date), factor_symbol_ids_by_date: Arc::new(factor_symbol_ids_by_date),
factor_text_by_date, factor_text_by_date,
factor_text_index, factor_text_index,
candidate_by_date, candidate_by_date,
candidate_row_index_by_date: Arc::new(candidate_row_index_by_date), candidate_symbol_ids_by_date: Arc::new(candidate_symbol_ids_by_date),
corporate_actions_by_date, corporate_actions_by_date,
execution_quotes_by_date, execution_quotes_by_date,
order_book_depth_index, order_book_depth_index,
@@ -1500,9 +1465,9 @@ impl DataSet {
date: NaiveDate, date: NaiveDate,
symbol_id: u32, symbol_id: u32,
) -> Option<&DailyMarketSnapshot> { ) -> Option<&DailyMarketSnapshot> {
find_arc_by_dense_row_index( find_arc_by_symbol_id(
self.market_by_date.get(&date)?, self.market_by_date.get(&date)?,
self.market_row_index_by_date.get(&date)?, self.market_symbol_ids_by_date.get(&date)?,
symbol_id, symbol_id,
) )
} }
@@ -1517,15 +1482,6 @@ impl DataSet {
.as_deref() .as_deref()
} }
pub fn market_rolling_cursor_by_symbol_id(
&self,
date: NaiveDate,
symbol_id: u32,
) -> Option<MarketRollingCursor> {
self.market_series_by_symbol_id(symbol_id)?
.rolling_cursor(date)
}
fn adjusted_close_series(&self, symbol: &str) -> Option<&AdjustedCloseSeries> { fn adjusted_close_series(&self, symbol: &str) -> Option<&AdjustedCloseSeries> {
self.adjusted_close_series_by_symbol self.adjusted_close_series_by_symbol
.get(symbol) .get(symbol)
@@ -1548,9 +1504,9 @@ impl DataSet {
date: NaiveDate, date: NaiveDate,
symbol_id: u32, symbol_id: u32,
) -> Option<&DailyFactorSnapshot> { ) -> Option<&DailyFactorSnapshot> {
find_arc_by_dense_row_index( find_arc_by_symbol_id(
self.factor_by_date.get(&date)?, self.factor_by_date.get(&date)?,
self.factor_row_index_by_date.get(&date)?, self.factor_symbol_ids_by_date.get(&date)?,
symbol_id, symbol_id,
) )
} }
@@ -1565,9 +1521,9 @@ impl DataSet {
date: NaiveDate, date: NaiveDate,
symbol_id: u32, symbol_id: u32,
) -> Option<&CandidateEligibility> { ) -> Option<&CandidateEligibility> {
find_arc_by_dense_row_index( find_arc_by_symbol_id(
self.candidate_by_date.get(&date)?, self.candidate_by_date.get(&date)?,
self.candidate_row_index_by_date.get(&date)?, self.candidate_symbol_ids_by_date.get(&date)?,
symbol_id, symbol_id,
) )
} }
@@ -2661,38 +2617,6 @@ impl DataSet {
} }
} }
pub fn market_decision_numeric_moving_average_at_cursor(
&self,
date: NaiveDate,
symbol_id: u32,
symbol: &str,
field: &str,
lookback: usize,
cursor: MarketRollingCursor,
) -> Option<f64> {
let field = normalized_field(field);
match field.as_ref() {
"close" | "prev_close" | "stock_close" | "price" => self
.adjusted_close_series_by_symbol_id(symbol_id)
.and_then(|series| {
series.decision_moving_average_at_end(cursor.decision_end, lookback)
}),
"volume" | "stock_volume" => {
self.market_series_by_symbol_id(symbol_id)
.and_then(|series| {
series.decision_volume_moving_average_at_end(cursor.decision_end, lookback)
})
}
_ => self.market_decision_numeric_moving_average_by_symbol_id(
date,
symbol_id,
symbol,
field.as_ref(),
lookback,
),
}
}
pub fn market_current_numeric_moving_average( pub fn market_current_numeric_moving_average(
&self, &self,
date: NaiveDate, date: NaiveDate,
@@ -2748,38 +2672,6 @@ impl DataSet {
} }
} }
pub fn market_current_numeric_moving_average_at_cursor(
&self,
date: NaiveDate,
symbol_id: u32,
symbol: &str,
field: &str,
lookback: usize,
cursor: MarketRollingCursor,
) -> Option<f64> {
let field = normalized_field(field);
match field.as_ref() {
"close" | "prev_close" | "stock_close" | "price" => self
.adjusted_close_series_by_symbol_id(symbol_id)
.and_then(|series| {
series.current_moving_average_at_end(cursor.current_end, lookback)
}),
"volume" | "stock_volume" => {
self.market_series_by_symbol_id(symbol_id)
.and_then(|series| {
series.current_volume_moving_average_at_end(cursor.current_end, lookback)
})
}
_ => self.market_current_numeric_moving_average_by_symbol_id(
date,
symbol_id,
symbol,
field.as_ref(),
lookback,
),
}
}
pub fn market_latest_back_adjusted_close(&self, date: NaiveDate, symbol: &str) -> Option<f64> { pub fn market_latest_back_adjusted_close(&self, date: NaiveDate, symbol: &str) -> Option<f64> {
self.adjusted_close_series(symbol) self.adjusted_close_series(symbol)
.and_then(|series| series.latest_back_adjusted_close(date)) .and_then(|series| series.latest_back_adjusted_close(date))
@@ -3468,40 +3360,17 @@ where
.collect() .collect()
} }
fn build_group_row_indices<T, F>( fn find_arc_by_symbol_id<'a, T>(
groups: &BTreeMap<NaiveDate, Vec<Arc<T>>>,
symbol_id_by_code: &HashMap<String, u32>,
symbol_of: F,
) -> BTreeMap<NaiveDate, Vec<u32>>
where
F: Fn(&T) -> &str + Copy,
{
groups
.iter()
.map(|(date, rows)| {
let mut row_indices = vec![u32::MAX; symbol_id_by_code.len()];
for (row_index, row) in rows.iter().enumerate() {
let symbol_id = *symbol_id_by_code
.get(symbol_of(row.as_ref()))
.expect("snapshot symbol missing from FIDC symbol index");
let slot = &mut row_indices[symbol_id as usize];
debug_assert_eq!(*slot, u32::MAX, "duplicate symbol in daily snapshot rows");
*slot = u32::try_from(row_index)
.expect("daily snapshot row index exceeds u32 capacity");
}
(*date, row_indices)
})
.collect()
}
fn find_arc_by_dense_row_index<'a, T>(
rows: &'a [Arc<T>], rows: &'a [Arc<T>],
row_indices: &[u32], symbol_ids: &[u32],
symbol_id: u32, symbol_id: u32,
) -> Option<&'a T> { ) -> Option<&'a T> {
let row_index = *row_indices.get(symbol_id as usize)?; if rows.len() != symbol_ids.len() {
(row_index != u32::MAX) return None;
.then_some(row_index as usize) }
symbol_ids
.binary_search(&symbol_id)
.ok()
.and_then(|index| rows.get(index)) .and_then(|index| rows.get(index))
.map(Arc::as_ref) .map(Arc::as_ref)
} }
crates/fidc-core/src/platform_expr_strategy.rs
+7 -45
View File
@@ -8,8 +8,8 @@ use rhai::{AST, Dynamic, Engine, Map, Scope};
use crate::broker::{MatchingType, RebalanceCashMode, SlippageModel}; use crate::broker::{MatchingType, RebalanceCashMode, SlippageModel};
use crate::cost::ChinaAShareCostModel; use crate::cost::ChinaAShareCostModel;
use crate::data::{ use crate::data::{
DailyMarketSnapshot, EligibleUniverseSnapshot, MarketRollingCursor, PriceField, DailyMarketSnapshot, EligibleUniverseSnapshot, PriceField, decision_free_float_cap_bn,
decision_free_float_cap_bn, decision_market_cap_bn, decision_market_cap_bn,
}; };
use crate::engine::BacktestError; use crate::engine::BacktestError;
use crate::events::OrderSide; use crate::events::OrderSide;
@@ -616,8 +616,6 @@ struct DayExpressionState {
struct StockExpressionState { struct StockExpressionState {
symbol: String, symbol: String,
symbol_id: u32, symbol_id: u32,
rolling_date: NaiveDate,
rolling_cursor: MarketRollingCursor,
market_cap: f64, market_cap: f64,
market_cap_bn: f64, market_cap_bn: f64,
free_float_cap: f64, free_float_cap: f64,
@@ -3616,21 +3614,14 @@ impl PlatformExprStrategy {
ctx: &StrategyContext<'_>, ctx: &StrategyContext<'_>,
date: NaiveDate, date: NaiveDate,
symbol_id: u32, symbol_id: u32,
rolling_cursor: Option<MarketRollingCursor>,
symbol: &str, symbol: &str,
field: &str, field: &str,
lookback: usize, lookback: usize,
) -> Option<f64> { ) -> Option<f64> {
match rolling_cursor { ctx.data
Some(cursor) => ctx.data.market_decision_numeric_moving_average_at_cursor(
date, symbol_id, symbol, field, lookback, cursor,
),
None => ctx
.data
.market_decision_numeric_moving_average_by_symbol_id( .market_decision_numeric_moving_average_by_symbol_id(
date, symbol_id, symbol, field, lookback, date, symbol_id, symbol, field, lookback,
), )
}
} }
fn stock_current_rolling_mean( fn stock_current_rolling_mean(
@@ -3638,19 +3629,13 @@ impl PlatformExprStrategy {
ctx: &StrategyContext<'_>, ctx: &StrategyContext<'_>,
date: NaiveDate, date: NaiveDate,
symbol_id: u32, symbol_id: u32,
rolling_cursor: Option<MarketRollingCursor>,
symbol: &str, symbol: &str,
field: &str, field: &str,
lookback: usize, lookback: usize,
) -> Option<f64> { ) -> Option<f64> {
match rolling_cursor { ctx.data.market_current_numeric_moving_average_by_symbol_id(
Some(cursor) => ctx.data.market_current_numeric_moving_average_at_cursor(
date, symbol_id, symbol, field, lookback, cursor,
),
None => ctx.data.market_current_numeric_moving_average_by_symbol_id(
date, symbol_id, symbol, field, lookback, date, symbol_id, symbol, field, lookback,
), )
}
} }
fn stock_state_at_time( fn stock_state_at_time(
@@ -3731,16 +3716,6 @@ impl PlatformExprStrategy {
symbol: symbol.to_string(), symbol: symbol.to_string(),
}) })
})?; })?;
let rolling_cursor = ctx
.data
.market_rolling_cursor_by_symbol_id(date, symbol_id)
.ok_or_else(|| {
BacktestError::Data(crate::data::DataSetError::MissingSnapshot {
kind: "market rolling cursor",
date,
symbol: symbol.to_string(),
})
})?;
{ {
let mut cache_date = self.stock_state_cache_date.borrow_mut(); let mut cache_date = self.stock_state_cache_date.borrow_mut();
if *cache_date != Some(date) { if *cache_date != Some(date) {
@@ -3785,15 +3760,7 @@ impl PlatformExprStrategy {
if !self.stock_rolling_requirements.requires(field, lookback) { if !self.stock_rolling_requirements.requires(field, lookback) {
return f64::NAN; return f64::NAN;
} }
self.stock_decision_rolling_mean( self.stock_decision_rolling_mean(ctx, date, symbol_id, symbol, field, lookback)
ctx,
date,
symbol_id,
Some(rolling_cursor),
symbol,
field,
lookback,
)
.unwrap_or(f64::NAN) .unwrap_or(f64::NAN)
}; };
let stock_ma_short = rolling("close", self.config.stock_short_ma_days); let stock_ma_short = rolling("close", self.config.stock_short_ma_days);
@@ -3920,8 +3887,6 @@ impl PlatformExprStrategy {
let state = StockExpressionState { let state = StockExpressionState {
symbol: symbol.to_string(), symbol: symbol.to_string(),
symbol_id, symbol_id,
rolling_date: date,
rolling_cursor,
market_cap, market_cap,
market_cap_bn, market_cap_bn,
free_float_cap, free_float_cap,
@@ -5409,7 +5374,6 @@ impl PlatformExprStrategy {
ctx, ctx,
day.date, day.date,
stock.symbol_id, stock.symbol_id,
(stock.rolling_date == day.date).then_some(stock.rolling_cursor),
&stock.symbol, &stock.symbol,
field, field,
lookback, lookback,
@@ -6074,7 +6038,6 @@ impl PlatformExprStrategy {
ctx, ctx,
day.date, day.date,
stock.symbol_id, stock.symbol_id,
(stock.rolling_date == day.date).then_some(stock.rolling_cursor),
&stock.symbol, &stock.symbol,
other, other,
lookback, lookback,
@@ -6124,7 +6087,6 @@ impl PlatformExprStrategy {
ctx, ctx,
day.date, day.date,
stock.symbol_id, stock.symbol_id,
(stock.rolling_date == day.date).then_some(stock.rolling_cursor),
&stock.symbol, &stock.symbol,
other, other,
lookback, lookback,