From 9a54156df94cfbf11a1e6335ec6ef5449bd6ac17 Mon Sep 17 00:00:00 2001 From: boris Date: Mon, 14 Sep 2026 04:00:39 +0800 Subject: [PATCH] docs(data): record DayOpen correctness and exact-column memory acceptance --- .../acceptance.json | 648 ++++++++++++++++++ ...ies-column-storage-performance-20260914.md | 117 ++++ 2 files changed, 765 insertions(+) create mode 100644 docs/evidence/series-column-storage-20260914/acceptance.json create mode 100644 docs/series-column-storage-performance-20260914.md diff --git a/docs/evidence/series-column-storage-20260914/acceptance.json b/docs/evidence/series-column-storage-20260914/acceptance.json new file mode 100644 index 0000000..f34bb30 --- /dev/null +++ b/docs/evidence/series-column-storage-20260914/acceptance.json @@ -0,0 +1,648 @@ +{ + "schema": "fidc.series-column-storage-acceptance/v1", + "rows": [ + { + "name": "control-1", + "receiptSha256": "fa8f2265f1d3ed2cdeef02f62840fec00a266ace3cf9399cb81d33110f3b115c", + "wallSeconds": 23.878584733000025, + "engineSeconds": 6.612, + "dataSeconds": 5.213, + "datasetConstructSeconds": 1.901, + "loopSeconds": 1.734, + "validationSeconds": 10.871, + "resultSeconds": 1.007, + "maxRssKiB": 7137676, + "canonical": { + "schemaVersion": "fidc-canonical-backtest-result/v2", + "algorithm": "sha256", + "ordering": "engine_fact_order_v2", + "totalRows": 93895, + "sha256": "3f089cbcc5412e07bbe2308d0dd60ac561a119f0a2228010fc3323714fde8ca7", + "sections": { + "accountEvents": { + "rowCount": 21555, + "sha256": "8c839c89191d2b5220fa1dd86f8d74bdd57fa566a096a0bbf6932d0247b48e8a" + }, + "equityFacts": { + "rowCount": 1025, + "sha256": "f928695650bdd90eb25d2acd478e0064046883649efa25cd2d4cdbbd27035c27" + }, + "fillEvents": { + "rowCount": 21393, + "sha256": "2a90bef0994dda1b7f6e914e4c58037ec0b91cd828eb09745f71aecde071f791" + }, + "holdingSnapshots": { + "rowCount": 28353, + "sha256": "799371917e516fb251b4afb60aeb4a1b8fc4b269ecfc27f827a3b388c4985a04" + }, + "orderEvents": { + "rowCount": 21491, + "sha256": "0685ac0c31933b6cf5d1319912de3f398c40ce7ee01c28b4abec1b98f92fb318" + }, + "riskAudits": { + "rowCount": 78, + "sha256": "e394cc9e8f3bac17a2f80f9db8738b91d4e960ff230647e17e5145b86d14b986" + } + } + }, + "storeSha256": "1905f0c8a887215279342b26d5769a6cbe40058971eef15adf86ef2eaa02aeb9", + "verifiedFactBlocks": 290, + "source": { + "commit": "d5b682c6d097", + "pid": 1700096, + "source_stale": false + } + }, + { + "name": "candidate-1", + "receiptSha256": "edd7e575013932ae58603e9fa810626573e1b0eaa00d01426d736cb4740aaf86", + "wallSeconds": 24.12557172914967, + "engineSeconds": 6.674, + "dataSeconds": 4.818, + "datasetConstructSeconds": 1.589, + "loopSeconds": 1.646, + "validationSeconds": 11.47, + "resultSeconds": 1.011, + "maxRssKiB": 6463660, + "canonical": { + "schemaVersion": "fidc-canonical-backtest-result/v2", + "algorithm": "sha256", + "ordering": "engine_fact_order_v2", + "totalRows": 93895, + "sha256": "3f089cbcc5412e07bbe2308d0dd60ac561a119f0a2228010fc3323714fde8ca7", + "sections": { + "accountEvents": { + "rowCount": 21555, + "sha256": "8c839c89191d2b5220fa1dd86f8d74bdd57fa566a096a0bbf6932d0247b48e8a" + }, + "equityFacts": { + "rowCount": 1025, + "sha256": "f928695650bdd90eb25d2acd478e0064046883649efa25cd2d4cdbbd27035c27" + }, + "fillEvents": { + "rowCount": 21393, + "sha256": "2a90bef0994dda1b7f6e914e4c58037ec0b91cd828eb09745f71aecde071f791" + }, + "holdingSnapshots": { + "rowCount": 28353, + "sha256": "799371917e516fb251b4afb60aeb4a1b8fc4b269ecfc27f827a3b388c4985a04" + }, + "orderEvents": { + "rowCount": 21491, + "sha256": "0685ac0c31933b6cf5d1319912de3f398c40ce7ee01c28b4abec1b98f92fb318" + }, + "riskAudits": { + "rowCount": 78, + "sha256": "e394cc9e8f3bac17a2f80f9db8738b91d4e960ff230647e17e5145b86d14b986" + } + } + }, + "storeSha256": "1905f0c8a887215279342b26d5769a6cbe40058971eef15adf86ef2eaa02aeb9", + "verifiedFactBlocks": 290, + "source": { + "commit": "d5b682c6d097", + "pid": 1700096, + "source_stale": false + } + }, + { + "name": "rotation-control-2", + "receiptSha256": "6a5b37c2881c7a4177ce69cef2440c3dd7a0846c6ca9a4811c68f173b16f7183", + "wallSeconds": 18.18023332185112, + "engineSeconds": 9.509, + "dataSeconds": 7.272, + "datasetConstructSeconds": 2.824, + "loopSeconds": 2.822, + "validationSeconds": 0.005, + "resultSeconds": 1.206, + "maxRssKiB": 7138628, + "canonical": { + "schemaVersion": "fidc-canonical-backtest-result/v2", + "algorithm": "sha256", + "ordering": "engine_fact_order_v2", + "totalRows": 93895, + "sha256": "3f089cbcc5412e07bbe2308d0dd60ac561a119f0a2228010fc3323714fde8ca7", + "sections": { + "accountEvents": { + "rowCount": 21555, + "sha256": "8c839c89191d2b5220fa1dd86f8d74bdd57fa566a096a0bbf6932d0247b48e8a" + }, + "equityFacts": { + "rowCount": 1025, + "sha256": "f928695650bdd90eb25d2acd478e0064046883649efa25cd2d4cdbbd27035c27" + }, + "fillEvents": { + "rowCount": 21393, + "sha256": "2a90bef0994dda1b7f6e914e4c58037ec0b91cd828eb09745f71aecde071f791" + }, + "holdingSnapshots": { + "rowCount": 28353, + "sha256": "799371917e516fb251b4afb60aeb4a1b8fc4b269ecfc27f827a3b388c4985a04" + }, + "orderEvents": { + "rowCount": 21491, + "sha256": "0685ac0c31933b6cf5d1319912de3f398c40ce7ee01c28b4abec1b98f92fb318" + }, + "riskAudits": { + "rowCount": 78, + "sha256": "e394cc9e8f3bac17a2f80f9db8738b91d4e960ff230647e17e5145b86d14b986" + } + } + }, + "storeSha256": "1905f0c8a887215279342b26d5769a6cbe40058971eef15adf86ef2eaa02aeb9", + "verifiedFactBlocks": 290, + "source": { + "commit": "d5b682c6d097", + "pid": 1700096, + "source_stale": false + } + }, + { + "name": "rotation-candidate-2", + "receiptSha256": "f50ca6af6496213d558af646862e5a86334077bb9080e4e7c6c9602a0cabd789", + "wallSeconds": 15.078223099000752, + "engineSeconds": 8.171, + "dataSeconds": 5.543, + "datasetConstructSeconds": 2.094, + "loopSeconds": 2.066, + "validationSeconds": 0.004, + "resultSeconds": 1.177, + "maxRssKiB": 6454624, + "canonical": { + "schemaVersion": "fidc-canonical-backtest-result/v2", + "algorithm": "sha256", + "ordering": "engine_fact_order_v2", + "totalRows": 93895, + "sha256": "3f089cbcc5412e07bbe2308d0dd60ac561a119f0a2228010fc3323714fde8ca7", + "sections": { + "accountEvents": { + "rowCount": 21555, + "sha256": "8c839c89191d2b5220fa1dd86f8d74bdd57fa566a096a0bbf6932d0247b48e8a" + }, + "equityFacts": { + "rowCount": 1025, + "sha256": "f928695650bdd90eb25d2acd478e0064046883649efa25cd2d4cdbbd27035c27" + }, + "fillEvents": { + "rowCount": 21393, + "sha256": "2a90bef0994dda1b7f6e914e4c58037ec0b91cd828eb09745f71aecde071f791" + }, + "holdingSnapshots": { + "rowCount": 28353, + "sha256": "799371917e516fb251b4afb60aeb4a1b8fc4b269ecfc27f827a3b388c4985a04" + }, + "orderEvents": { + "rowCount": 21491, + "sha256": "0685ac0c31933b6cf5d1319912de3f398c40ce7ee01c28b4abec1b98f92fb318" + }, + "riskAudits": { + "rowCount": 78, + "sha256": "e394cc9e8f3bac17a2f80f9db8738b91d4e960ff230647e17e5145b86d14b986" + } + } + }, + "storeSha256": "1905f0c8a887215279342b26d5769a6cbe40058971eef15adf86ef2eaa02aeb9", + "verifiedFactBlocks": 290, + "source": { + "commit": "d5b682c6d097", + "pid": 1700096, + "source_stale": false + } + }, + { + "name": "candidate-3", + "receiptSha256": "c8bce9a8eeda769f00e72118c9ea955d323fdad902dc271590d6e377b5a99196", + "wallSeconds": 13.124768079956993, + "engineSeconds": 6.649, + "dataSeconds": 5.103, + "datasetConstructSeconds": 1.853, + "loopSeconds": 1.676, + "validationSeconds": 0.208, + "resultSeconds": 1.017, + "maxRssKiB": 6457728, + "canonical": { + "schemaVersion": "fidc-canonical-backtest-result/v2", + "algorithm": "sha256", + "ordering": "engine_fact_order_v2", + "totalRows": 93895, + "sha256": "3f089cbcc5412e07bbe2308d0dd60ac561a119f0a2228010fc3323714fde8ca7", + "sections": { + "accountEvents": { + "rowCount": 21555, + "sha256": "8c839c89191d2b5220fa1dd86f8d74bdd57fa566a096a0bbf6932d0247b48e8a" + }, + "equityFacts": { + "rowCount": 1025, + "sha256": "f928695650bdd90eb25d2acd478e0064046883649efa25cd2d4cdbbd27035c27" + }, + "fillEvents": { + "rowCount": 21393, + "sha256": "2a90bef0994dda1b7f6e914e4c58037ec0b91cd828eb09745f71aecde071f791" + }, + "holdingSnapshots": { + "rowCount": 28353, + "sha256": "799371917e516fb251b4afb60aeb4a1b8fc4b269ecfc27f827a3b388c4985a04" + }, + "orderEvents": { + "rowCount": 21491, + "sha256": "0685ac0c31933b6cf5d1319912de3f398c40ce7ee01c28b4abec1b98f92fb318" + }, + "riskAudits": { + "rowCount": 78, + "sha256": "e394cc9e8f3bac17a2f80f9db8738b91d4e960ff230647e17e5145b86d14b986" + } + } + }, + "storeSha256": "1905f0c8a887215279342b26d5769a6cbe40058971eef15adf86ef2eaa02aeb9", + "verifiedFactBlocks": 290, + "source": { + "commit": "d5b682c6d097", + "pid": 1700096, + "source_stale": false + } + }, + { + "name": "control-3", + "receiptSha256": "1f999a89aadab1baf4a70cda96a314ab237302722cdce895bb5fe26dea4306ec", + "wallSeconds": 12.725315875839442, + "engineSeconds": 6.583, + "dataSeconds": 4.955, + "datasetConstructSeconds": 1.713, + "loopSeconds": 1.675, + "validationSeconds": 0.004, + "resultSeconds": 1.013, + "maxRssKiB": 7140772, + "canonical": { + "schemaVersion": "fidc-canonical-backtest-result/v2", + "algorithm": "sha256", + "ordering": "engine_fact_order_v2", + "totalRows": 93895, + "sha256": "3f089cbcc5412e07bbe2308d0dd60ac561a119f0a2228010fc3323714fde8ca7", + "sections": { + "accountEvents": { + "rowCount": 21555, + "sha256": "8c839c89191d2b5220fa1dd86f8d74bdd57fa566a096a0bbf6932d0247b48e8a" + }, + "equityFacts": { + "rowCount": 1025, + "sha256": "f928695650bdd90eb25d2acd478e0064046883649efa25cd2d4cdbbd27035c27" + }, + "fillEvents": { + "rowCount": 21393, + "sha256": "2a90bef0994dda1b7f6e914e4c58037ec0b91cd828eb09745f71aecde071f791" + }, + "holdingSnapshots": { + "rowCount": 28353, + "sha256": "799371917e516fb251b4afb60aeb4a1b8fc4b269ecfc27f827a3b388c4985a04" + }, + "orderEvents": { + "rowCount": 21491, + "sha256": "0685ac0c31933b6cf5d1319912de3f398c40ce7ee01c28b4abec1b98f92fb318" + }, + "riskAudits": { + "rowCount": 78, + "sha256": "e394cc9e8f3bac17a2f80f9db8738b91d4e960ff230647e17e5145b86d14b986" + } + } + }, + "storeSha256": "1905f0c8a887215279342b26d5769a6cbe40058971eef15adf86ef2eaa02aeb9", + "verifiedFactBlocks": 290, + "source": { + "commit": "d5b682c6d097", + "pid": 1700096, + "source_stale": false + } + }, + { + "name": "trend-40-control", + "receiptSha256": "55b1976d15ba531477ff92f07f953a0a8a9b7cab96f01fb0ce5546f9aeda7406", + "wallSeconds": 14.779385674046353, + "engineSeconds": 7.867, + "dataSeconds": 5.418, + "datasetConstructSeconds": 1.921, + "loopSeconds": 1.952, + "validationSeconds": 0.005, + "resultSeconds": 1.35, + "maxRssKiB": 7157844, + "canonical": { + "schemaVersion": "fidc-canonical-backtest-result/v2", + "algorithm": "sha256", + "ordering": "engine_fact_order_v2", + "totalRows": 128192, + "sha256": "41209fed4c7a5e385a12e170afef685efae7a3b00137365fe2b9aea015dd7948", + "sections": { + "accountEvents": { + "rowCount": 29968, + "sha256": "fa578b86d94a5be9ad192258eb87c9be6a1b356d6713e70ae475b95fd130f61d" + }, + "equityFacts": { + "rowCount": 1025, + "sha256": "90f63f3c51c62f0fc0abb317a29ac48aa0ead6035d5f75d650a948c33ae1e9b3" + }, + "fillEvents": { + "rowCount": 29776, + "sha256": "6626ea78cfaa5d88040496c575be63cd8b573d48d1afe6812579762bf7e2abd0" + }, + "holdingSnapshots": { + "rowCount": 37367, + "sha256": "491b32468148cc62779f4c01cee96e4dcf18ce1a6115ccecc437b468e49b2ceb" + }, + "orderEvents": { + "rowCount": 29932, + "sha256": "565f5a6271a9e1f0fa31f152141c75db0e5f84784e4d4a83b2277b0fd14c45fc" + }, + "riskAudits": { + "rowCount": 124, + "sha256": "d170e1704ee5f64a93b71cecf2178b062d30368df8f5da53c07416b8e8a9c5e5" + } + } + }, + "storeSha256": "ab29a9f999a6b41330255ba82081696a9826825fa6d34941247332adac65249e", + "verifiedFactBlocks": 293, + "source": { + "commit": "d5b682c6d097", + "pid": 1700096, + "source_stale": false + } + }, + { + "name": "trend-40-candidate", + "receiptSha256": "86a3f452168d059f034e91f7317890af49d9763b75745dd3d39a948b00c1d72d", + "wallSeconds": 15.011793529149145, + "engineSeconds": 8.011, + "dataSeconds": 5.168, + "datasetConstructSeconds": 1.851, + "loopSeconds": 1.739, + "validationSeconds": 0.376, + "resultSeconds": 1.315, + "maxRssKiB": 6470768, + "canonical": { + "schemaVersion": "fidc-canonical-backtest-result/v2", + "algorithm": "sha256", + "ordering": "engine_fact_order_v2", + "totalRows": 128192, + "sha256": "41209fed4c7a5e385a12e170afef685efae7a3b00137365fe2b9aea015dd7948", + "sections": { + "accountEvents": { + "rowCount": 29968, + "sha256": "fa578b86d94a5be9ad192258eb87c9be6a1b356d6713e70ae475b95fd130f61d" + }, + "equityFacts": { + "rowCount": 1025, + "sha256": "90f63f3c51c62f0fc0abb317a29ac48aa0ead6035d5f75d650a948c33ae1e9b3" + }, + "fillEvents": { + "rowCount": 29776, + "sha256": "6626ea78cfaa5d88040496c575be63cd8b573d48d1afe6812579762bf7e2abd0" + }, + "holdingSnapshots": { + "rowCount": 37367, + "sha256": "491b32468148cc62779f4c01cee96e4dcf18ce1a6115ccecc437b468e49b2ceb" + }, + "orderEvents": { + "rowCount": 29932, + "sha256": "565f5a6271a9e1f0fa31f152141c75db0e5f84784e4d4a83b2277b0fd14c45fc" + }, + "riskAudits": { + "rowCount": 124, + "sha256": "d170e1704ee5f64a93b71cecf2178b062d30368df8f5da53c07416b8e8a9c5e5" + } + } + }, + "storeSha256": "ab29a9f999a6b41330255ba82081696a9826825fa6d34941247332adac65249e", + "verifiedFactBlocks": 293, + "source": { + "commit": "d5b682c6d097", + "pid": 1700096, + "source_stale": false + } + }, + { + "name": "pullback-40-control", + "receiptSha256": "f5e6c9cb90db0dc7095e388b4ee939d4c6609824f3c065e0a08144a80f3d9395", + "wallSeconds": 14.010676869889721, + "engineSeconds": 7.298, + "dataSeconds": 5.172, + "datasetConstructSeconds": 1.893, + "loopSeconds": 1.7, + "validationSeconds": 0.005, + "resultSeconds": 1.388, + "maxRssKiB": 7167408, + "canonical": { + "schemaVersion": "fidc-canonical-backtest-result/v2", + "algorithm": "sha256", + "ordering": "engine_fact_order_v2", + "totalRows": 135630, + "sha256": "49fdbd74245d7aa678f1a4851add29f7b12dde71fd50c93c070fd2ee57f35285", + "sections": { + "accountEvents": { + "rowCount": 32010, + "sha256": "9be3914d28d7766f12bac45227d2c2da47d61a921f9b918a4b11447da4b78baa" + }, + "equityFacts": { + "rowCount": 1025, + "sha256": "2044627b1152c4a2ad8ea92dca3351322d41cfc18c002cda93f274bd810efdad" + }, + "fillEvents": { + "rowCount": 31862, + "sha256": "b33d1b4b0f7f86c96c082c217c9a27d86830ccac0de5a33714104ac4756df8d7" + }, + "holdingSnapshots": { + "rowCount": 38679, + "sha256": "a45b516a926c57b6c7827f8f1684b6e749d26118b8c5d8e8614afbb23cd87559" + }, + "orderEvents": { + "rowCount": 31966, + "sha256": "73704496ab17d10d9a602429d97b728cf22d0896f2d467b0f2064f2f48bf04fb" + }, + "riskAudits": { + "rowCount": 88, + "sha256": "2c8de7dbb11c7ca5ac678c17feabec0bd98aac0a031bbb4780eddaa144933293" + } + } + }, + "storeSha256": "ff32f177e5e0ec4b4f3f0597b61174efdafac77b4f36e6065448b6e9c414f07e", + "verifiedFactBlocks": 281, + "source": { + "commit": "d5b682c6d097", + "pid": 1700096, + "source_stale": false + } + }, + { + "name": "pullback-40-candidate", + "receiptSha256": "f8caf67a6010d882a064678cf5c57f48c53c85b7c82dde708c029ee52e9e56b2", + "wallSeconds": 13.87669027899392, + "engineSeconds": 7.219, + "dataSeconds": 5.117, + "datasetConstructSeconds": 1.837, + "loopSeconds": 1.691, + "validationSeconds": 0.005, + "resultSeconds": 1.362, + "maxRssKiB": 6478492, + "canonical": { + "schemaVersion": "fidc-canonical-backtest-result/v2", + "algorithm": "sha256", + "ordering": "engine_fact_order_v2", + "totalRows": 135630, + "sha256": "49fdbd74245d7aa678f1a4851add29f7b12dde71fd50c93c070fd2ee57f35285", + "sections": { + "accountEvents": { + "rowCount": 32010, + "sha256": "9be3914d28d7766f12bac45227d2c2da47d61a921f9b918a4b11447da4b78baa" + }, + "equityFacts": { + "rowCount": 1025, + "sha256": "2044627b1152c4a2ad8ea92dca3351322d41cfc18c002cda93f274bd810efdad" + }, + "fillEvents": { + "rowCount": 31862, + "sha256": "b33d1b4b0f7f86c96c082c217c9a27d86830ccac0de5a33714104ac4756df8d7" + }, + "holdingSnapshots": { + "rowCount": 38679, + "sha256": "a45b516a926c57b6c7827f8f1684b6e749d26118b8c5d8e8614afbb23cd87559" + }, + "orderEvents": { + "rowCount": 31966, + "sha256": "73704496ab17d10d9a602429d97b728cf22d0896f2d467b0f2064f2f48bf04fb" + }, + "riskAudits": { + "rowCount": 88, + "sha256": "2c8de7dbb11c7ca5ac678c17feabec0bd98aac0a031bbb4780eddaa144933293" + } + } + }, + "storeSha256": "ff32f177e5e0ec4b4f3f0597b61174efdafac77b4f36e6065448b6e9c414f07e", + "verifiedFactBlocks": 281, + "source": { + "commit": "d5b682c6d097", + "pid": 1700096, + "source_stale": false + } + }, + { + "name": "volume-momentum-80-control", + "receiptSha256": "0f589d11fec4a37f80635446fa445b7c7a9ae58e202e5b0c2533aa257fd94c81", + "wallSeconds": 18.577259425073862, + "engineSeconds": 10.96, + "dataSeconds": 5.163, + "datasetConstructSeconds": 1.891, + "loopSeconds": 1.698, + "validationSeconds": 0.004, + "resultSeconds": 2.282, + "maxRssKiB": 7210220, + "canonical": { + "schemaVersion": "fidc-canonical-backtest-result/v2", + "algorithm": "sha256", + "ordering": "engine_fact_order_v2", + "totalRows": 234267, + "sha256": "4359817bb1fbd73b02c2c3456e1f0b7ee7428b090529393bd203e8212979a1dc", + "sections": { + "accountEvents": { + "rowCount": 51696, + "sha256": "ca4d333cc4768ec4b528475d1833809c7e76a80de937c635e7e933a0f18264c6" + }, + "equityFacts": { + "rowCount": 1025, + "sha256": "31865ccfeb71d260757979524a679880070d691c09083c82865c3de6dd47e440" + }, + "fillEvents": { + "rowCount": 51300, + "sha256": "4f45cc0906b6cd02af9ce12450a52f509f6e80e9b26157695ae04e90ce7ca9ac" + }, + "holdingSnapshots": { + "rowCount": 78078, + "sha256": "79098ebfa48dcd24b00ba4d19917c291adb80a129b4d90ee1087b5f2210c43aa" + }, + "orderEvents": { + "rowCount": 51783, + "sha256": "340699c5d04407c5c6f71dc39c41ac2baedc096bb11185f1ccb99ab7100ba8f8" + }, + "riskAudits": { + "rowCount": 385, + "sha256": "2aa0a4d6e6078dbea815b536ce0ee8f6ca03f9c44f63cdfecf4e37a0ff9cf561" + } + } + }, + "storeSha256": "5748fe4db08f8d81607622b302cc43c1d2e4ef50c3b87739fb6c20cd40de79c0", + "verifiedFactBlocks": 309, + "source": { + "commit": "d5b682c6d097", + "pid": 1700096, + "source_stale": false + } + }, + { + "name": "volume-momentum-80-candidate", + "receiptSha256": "5ae74a1b311479a39c9863c2fba487951631353f9ae2a523d919f2d0a8f592e7", + "wallSeconds": 18.476340716006234, + "engineSeconds": 10.989, + "dataSeconds": 5.051, + "datasetConstructSeconds": 1.844, + "loopSeconds": 1.645, + "validationSeconds": 0.005, + "resultSeconds": 2.284, + "maxRssKiB": 6527236, + "canonical": { + "schemaVersion": "fidc-canonical-backtest-result/v2", + "algorithm": "sha256", + "ordering": "engine_fact_order_v2", + "totalRows": 234267, + "sha256": "4359817bb1fbd73b02c2c3456e1f0b7ee7428b090529393bd203e8212979a1dc", + "sections": { + "accountEvents": { + "rowCount": 51696, + "sha256": "ca4d333cc4768ec4b528475d1833809c7e76a80de937c635e7e933a0f18264c6" + }, + "equityFacts": { + "rowCount": 1025, + "sha256": "31865ccfeb71d260757979524a679880070d691c09083c82865c3de6dd47e440" + }, + "fillEvents": { + "rowCount": 51300, + "sha256": "4f45cc0906b6cd02af9ce12450a52f509f6e80e9b26157695ae04e90ce7ca9ac" + }, + "holdingSnapshots": { + "rowCount": 78078, + "sha256": "79098ebfa48dcd24b00ba4d19917c291adb80a129b4d90ee1087b5f2210c43aa" + }, + "orderEvents": { + "rowCount": 51783, + "sha256": "340699c5d04407c5c6f71dc39c41ac2baedc096bb11185f1ccb99ab7100ba8f8" + }, + "riskAudits": { + "rowCount": 385, + "sha256": "2aa0a4d6e6078dbea815b536ce0ee8f6ca03f9c44f63cdfecf4e37a0ff9cf561" + } + } + }, + "storeSha256": "5748fe4db08f8d81607622b302cc43c1d2e4ef50c3b87739fb6c20cd40de79c0", + "verifiedFactBlocks": 309, + "source": { + "commit": "d5b682c6d097", + "pid": 1700096, + "source_stale": false + } + } + ], + "sharedInputFiles": 9257, + "sharedInputBytes": 12596608049, + "sharedInputInventorySha256": "1a4818aaab906e77b750e28601d3d405ad9e14e0553f7937cc60b68be0c9b71d", + "verifiedFactBlocks": 3506, + "status": "candidate-not-deployed", + "scope": "exact in-memory column reuse; separate DayOpen correctness fix included in both control and candidate", + "controlRunnerSha256": "8859459f54389f12af1ab7d4e36802c01aff63fb10fbb679243ccdd54d013e2d", + "candidateRunnerSha256": "40bcf65c1977dbd93ab8bc80e3ff04d0db5e27b61fce1afdce99cf1b5e58eb43", + "candidateApiSha256": "c54be3a8196c32051520c709f793bcb974d869467bb12700d846efaad8c2180e", + "engineCommit": "996b909608589fb1987f33c0cfb4c62099f69617", + "serviceCommit": "443ed421c2c9c854a01fab69ce58957690504570", + "boundaries": [ + "The DayOpen prefix correction is present in both storage A/B binaries.", + "No file format, cache schema, input values or execution policy changed for the storage comparison.", + "Original shared inputs were hashed and remained unchanged; results were recalculated into private artifacts.", + "The last pair ran candidate before control. It did not establish a general latency improvement.", + "Source remains frozen and paused research/signal tasks were not resumed.", + "The independent same-day intraday clock counterexample remains unresolved." + ] +} diff --git a/docs/series-column-storage-performance-20260914.md b/docs/series-column-storage-performance-20260914.md new file mode 100644 index 0000000..c2dea1a --- /dev/null +++ b/docs/series-column-storage-performance-20260914.md @@ -0,0 +1,117 @@ +# Exact Series Column Storage + +## Status + +Candidate not deployed. Twelve real long replays preserve their independent +business baselines and reduce peak RSS by about 9.5%. Construction latency is +mixed, including a reversed pair where the control is faster. This is accepted +as evidence of a smaller working set, not as a proved general speedup or closure +of the main performance objective. Original results are retained unchanged. + +The Source implementation remains d5b682c6d09704ff23d725a8dd8b155db3eb6967. +Research/signal work remains paused. Current service remains the e81bf47/c98 +release; no Source, trading or other-user process was restarted or modified. +The independent intraday-clock counterexample remains open. + +## Separate DayOpen Correction + +Code inspection found that PriceField::DayOpen selected the Open prefix sums, +although direct history access returned day_open. For day_open values 10/12 +and open values 20/24, that path computes 22 instead of the expected 11. +The correction adds its own day-open prefix and a regression checking both +fields plus empty/insufficient windows. No price field is substituted. + +This correction was built and tested independently before the storage change: +806 core unit/integration tests, 448 runner tests and 119 API tests passed. +The resulting control runner is +8859459f54389f12af1ab7d4e36802c01aff63fb10fbb679243ccdd54d013e2d. +It also preserves the real rotation baseline. Both subsequent A/B variants +include the fix, so corrected calculation semantics are not counted as speedup. + +## Storage Design + +SymbolPriceSeries previously allocated separate vectors for last/bid/ask, +their prefix, timestamps, trading phases and three quote-volume fields, even +when actual data repeated or exactly matched the existing close series. + +- ReferenceMatchedValues aliases the existing column only after every consumed + value matches by f64::to_bits. A mismatch materializes the exact preceding + values and continues as an owned vector. No missing/invalid price is replaced + by close; signed zero and NaN payload differences prevent sharing. +- RepeatedValues retains the actual first value and logical length. It avoids + expanding equal values, including nonzero volumes and Some strings. The + first difference materializes the exact prior values. None is distinct from + an empty string; no value is inferred from the backtest frequency. +- Intraday updates materialize only changed columns. Cloned views retain their + original values and immutable daily base. Last-price prefix sums use the same + accumulation order and actual values as before. History cutoffs are unchanged. + +There is no new dependency, unsafe code, strategy-specific branch, disk schema, +source-data rewrite or account/result sharing. Construction and data validation +remain in the existing paths. The overlay comment now accurately states that +quote fields affect Last history while daily OHLC remains unchanged. + +The full candidate passes 813 core unit/integration tests (9 ignored), 448 runner +tests (9 ignored) and 119 API tests (5 ignored). New tests cover exact bit +identity, distinct zero/NaN values, repeated nonzero/string values, mutation +isolation, unknown dates, full snapshot equality and history-date cutoffs. +These tests do not prove the separately known same-day execution-clock issue. + +## Real A/B + +All cases execute 2021-08-23 through 2025-11-17 with their unchanged frozen +strategy/runtime/bundle and 10,000,000 initial cash. This is not five complete +execution years. Every run is a new process with private result artifacts and +the same verified shared inputs: 9,257 files / 12,596,608,049 bytes. No original +input changed and no Arrow/bin input was newly created. Hashing is outside the +runner timer; no result is reused. Source/OS caches are not cold. + +| Case | Wall s | Data s | DataSet construction s | Engine s | RSS KiB | +|---|---:|---:|---:|---:|---:| +| Rotation control 1 | 23.879 | 5.213 | 1.901 | 6.612 | 7,137,676 | +| Rotation candidate 1 | 24.126 | 4.818 | 1.589 | 6.674 | 6,463,660 | +| Rotation control 2 | 18.180 | 7.272 | 2.824 | 9.509 | 7,138,628 | +| Rotation candidate 2 | 15.078 | 5.543 | 2.094 | 8.171 | 6,454,624 | +| Rotation candidate 3 | 13.125 | 5.103 | 1.853 | 6.649 | 6,457,728 | +| Rotation control 3 | 12.725 | 4.955 | 1.713 | 6.583 | 7,140,772 | +| Trend 40 control | 14.779 | 5.418 | 1.921 | 7.867 | 7,157,844 | +| Trend 40 candidate | 15.012 | 5.168 | 1.851 | 8.011 | 6,470,768 | +| Pullback 40 control | 14.011 | 5.172 | 1.893 | 7.298 | 7,167,408 | +| Pullback 40 candidate | 13.877 | 5.117 | 1.837 | 7.219 | 6,478,492 | +| Volume 80 control | 18.577 | 5.163 | 1.891 | 10.960 | 7,210,220 | +| Volume 80 candidate | 18.476 | 5.051 | 1.844 | 10.989 | 6,527,236 | + +The final rotation pair deliberately ran candidate before control. Rotation +RSS medians are 7,138,628 versus 6,457,728 KiB, about 665 MiB / 9.5% lower. +Other strategy pairs save about 670-673 MiB. These are measured process peaks, +not estimates obtained by adding cgroup limits or counting mmap as private RAM. + +Construction medians are 1.901 versus 1.853 seconds for rotation. The first pair +has a larger reduction, but other samples and the reversed pair do not support +a universal 16% construction or total-latency claim. Source validation waits and +independent phase variation remain in the full evidence. A read-only host sample +showed load near 49 and thermal readings 53/58/69 C; it does not prove the cause +of timing variation. No host policy or another user's workload was changed. + +All six canonical sections and result-store SHA match the appropriate existing +baselines: 21,393 / 29,776 / 31,862 / 51,300 fills. Result and request evidence, +physical manifests and all 3,506 fact blocks were verified. No earlier failed +or successful receipt was rewritten. Complete receipts remain on 177; only the +compact verified summary is stored here to avoid duplicating input inventories. + +## Remaining Work + +Do not publish this as the main performance fix. Next, target the remaining +daily snapshot/factor construction and direct typed-column reuse, avoiding +new per-access branches or post-hoc compression passes. Cold-query acceptance, +real minute-mode acceptance, HTTP publication and the intraday-clock issue are +still outstanding. Signal lifecycle, financial PIT and UI factor conditions +remain outside this completed storage experiment. + +- Engine candidate: 996b909608589fb1987f33c0cfb4c62099f69617. +- Service source: 443ed421c2c9c854a01fab69ce58957690504570. +- Candidate runner: 40bcf65c1977dbd93ab8bc80e3ff04d0db5e27b61fce1afdce99cf1b5e58eb43. +- Candidate API: c54be3a8196c32051520c709f793bcb974d869467bb12700d846efaad8c2180e. +- Evidence: /srv/fidc/canonical/run/research/series-column-storage-20260914. + +[Verified summary](evidence/series-column-storage-20260914/acceptance.json).