Revert "perf(data): sort compact market-cap keys without revisiting wide snapshots"
This reverts commit 4e953b6e98.
This commit is contained in:
boris
@@ -4644,23 +4644,25 @@ fn build_factor_market_cap_order(
|
|||||||
symbol_ids.len(),
|
symbol_ids.len(),
|
||||||
"factor rows and symbol ids diverged for {date}"
|
"factor rows and symbol ids diverged for {date}"
|
||||||
);
|
);
|
||||||
let mut keys = rows
|
let mut row_indices = rows
|
||||||
.iter()
|
.iter()
|
||||||
.zip(symbol_ids)
|
.enumerate()
|
||||||
.filter_map(|(row, symbol_id)| {
|
.filter_map(|(index, row)| {
|
||||||
let market_cap_bn = decision_market_cap_bn(row);
|
let market_cap_bn = decision_market_cap_bn(row);
|
||||||
(market_cap_bn.is_finite() && market_cap_bn > 0.0)
|
(market_cap_bn.is_finite() && market_cap_bn > 0.0).then_some(index)
|
||||||
.then_some((market_cap_bn, *symbol_id))
|
|
||||||
})
|
})
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
// Symbol IDs follow the complete lexical symbol order. Equal keys
|
row_indices.sort_by(|left, right| {
|
||||||
// emit identical IDs, so stability does not affect the output.
|
let left = &rows[*left];
|
||||||
keys.sort_unstable_by(|left, right| {
|
let right = &rows[*right];
|
||||||
left.0.total_cmp(&right.0).then_with(|| left.1.cmp(&right.1))
|
decision_market_cap_bn(left)
|
||||||
|
.partial_cmp(&decision_market_cap_bn(right))
|
||||||
|
.unwrap_or(std::cmp::Ordering::Equal)
|
||||||
|
.then_with(|| left.symbol.cmp(&right.symbol))
|
||||||
});
|
});
|
||||||
let ordered = keys
|
let ordered = row_indices
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|(_, symbol_id)| symbol_id)
|
.map(|index| symbol_ids[index])
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
(*date, ordered)
|
(*date, ordered)
|
||||||
})
|
})
|
||||||
@@ -5097,79 +5099,6 @@ fn instrument_passes_baseline_selection(instrument: Option<&Instrument>, date: N
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
fn reference_market_cap_order(
|
|
||||||
rows: &[DailyFactorSnapshot],
|
|
||||||
symbol_ids: &[u32],
|
|
||||||
) -> Vec<u32> {
|
|
||||||
let mut indices = (0..rows.len())
|
|
||||||
.filter(|index| {
|
|
||||||
let cap = decision_market_cap_bn(&rows[*index]);
|
|
||||||
cap.is_finite() && cap > 0.0
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
indices.sort_by(|left, right| {
|
|
||||||
decision_market_cap_bn(&rows[*left])
|
|
||||||
.partial_cmp(&decision_market_cap_bn(&rows[*right]))
|
|
||||||
.unwrap_or(std::cmp::Ordering::Equal)
|
|
||||||
.then_with(|| rows[*left].symbol.cmp(&rows[*right].symbol))
|
|
||||||
});
|
|
||||||
indices.into_iter().map(|index| symbol_ids[index]).collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn compact_market_cap_keys_preserve_reference_order_and_source_values() {
|
|
||||||
let start = NaiveDate::from_ymd_opt(2024, 1, 2).unwrap();
|
|
||||||
let caps = [
|
|
||||||
f64::NAN, -0.0, 0.0, f64::NEG_INFINITY, f64::INFINITY, -1.0,
|
|
||||||
f64::from_bits(1), f64::MIN_POSITIVE, 1.0, 1.0000000000000002,
|
|
||||||
1.0, f64::MAX,
|
|
||||||
];
|
|
||||||
let groups = (0..5).map(|day| {
|
|
||||||
let date = start + chrono::Duration::days(day);
|
|
||||||
let mut rows = (0..4096).map(|index| DailyFactorSnapshot {
|
|
||||||
date,
|
|
||||||
symbol: format!("instrument_{index:05}_long_non_equity_identity").into(),
|
|
||||||
market_cap_bn: caps[(index * 17 + day as usize) % caps.len()],
|
|
||||||
free_float_cap_bn: 1.0, pe_ttm: 1.0,
|
|
||||||
turnover_ratio: Some(-0.0), effective_turnover_ratio: None,
|
|
||||||
adjustment_factor_backward1: Some(1.0),
|
|
||||||
extra_factors: NumericFactorMap::new(),
|
|
||||||
}).collect::<Vec<_>>();
|
|
||||||
rows.reverse();
|
|
||||||
rows.push(rows[19].clone());
|
|
||||||
(date, rows)
|
|
||||||
}).chain([(start + chrono::Duration::days(5), Vec::new())])
|
|
||||||
.collect::<BTreeMap<_, _>>();
|
|
||||||
let symbol_index = build_symbol_id_index(
|
|
||||||
&HashMap::new(), &BTreeMap::new(), &groups, &BTreeMap::new(),
|
|
||||||
);
|
|
||||||
let ids = groups.iter().map(|(date, rows)| (*date, rows.iter()
|
|
||||||
.map(|row| symbol_index[row.symbol.as_str()]).collect::<Vec<_>>()))
|
|
||||||
.collect::<BTreeMap<_, _>>();
|
|
||||||
let expected = groups.iter().map(|(date, rows)|
|
|
||||||
(*date, reference_market_cap_order(rows, &ids[date])))
|
|
||||||
.collect::<BTreeMap<_, _>>();
|
|
||||||
let bits = || groups.values().flatten().map(|row|
|
|
||||||
(row.market_cap_bn.to_bits(), row.turnover_ratio.unwrap().to_bits()))
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
let before = bits();
|
|
||||||
for threads in [1, 2, 8] {
|
|
||||||
let pool = rayon::ThreadPoolBuilder::new().num_threads(threads).build().unwrap();
|
|
||||||
assert_eq!(pool.install(|| build_factor_market_cap_order(&groups, &ids)), expected);
|
|
||||||
assert_eq!(bits(), before);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
#[should_panic(expected = "factor rows and symbol ids diverged")]
|
|
||||||
fn compact_market_cap_keys_reject_misaligned_symbol_ids() {
|
|
||||||
let date = NaiveDate::from_ymd_opt(2024, 1, 2).unwrap();
|
|
||||||
build_factor_market_cap_order(
|
|
||||||
&BTreeMap::from([(date, Vec::new())]),
|
|
||||||
&BTreeMap::from([(date, vec![1])]),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn market_row(date: &str, prev_close: f64, volume: u64) -> DailyMarketSnapshot {
|
fn market_row(date: &str, prev_close: f64, volume: u64) -> DailyMarketSnapshot {
|
||||||
DailyMarketSnapshot {
|
DailyMarketSnapshot {
|
||||||
date: NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap(),
|
date: NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap(),
|
||||||
|
|||||||
Reference in New Issue
Block a user