Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cases/journey-3/J3-58/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "b0163f0cf1bf8be36dec6a6c726b75f1129ac83972e8024dade13b8e3df12e23"
"revision": "ops-20261118-r8",
"sha256": "efc069be8c22a9ecbd54ac621b1c15b5e9d67115181e712f4c36b85f8a74a545"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-59/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "f4187e4c6b9f9c40424c1b3c6f0527b44e9b40aa44629b691ac1a356da8364ec"
"revision": "ops-20261118-r8",
"sha256": "90739c2b4ac8f9251d3efbdf39e60195a296f70ced75b4ba52574dc39ad04b46"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-60/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r7",
"sha256": "7835c4ba6b2cea6a1919aa5b3586ab91070f48dbc8078da3b8377072b96c6f54"
"revision": "ops-20261118-r8",
"sha256": "305d319c6022f66092f23075b97da0dd462e5f98b6b19b2092dbc467b3264db9"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-61/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "7c154767f95a1b8875878ba7c817d42e00fca5bd13a7c5357a070a3fbf900e3b"
"revision": "ops-20261118-r8",
"sha256": "8cb6c029279dcfae84aafc2670697be546363fab91d3b1695b49a794b9cb24de"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-62/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "e91d1bff921d7b8ec765d8d687f169e9237470242f6e610408c3c4d46ff99466"
"revision": "ops-20261118-r8",
"sha256": "2828e97a23910dcfee587c4556aa126008b000ad9248fa6f10b336eb86a3cb73"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-63/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "cca8ab0bd282b11e8ba5a96827740065b37edc0c3598e549cc9acd92d974b1c9"
"revision": "ops-20261118-r8",
"sha256": "fa71854621b27f40e543c4045f21c0d0fd9b9b8a3c1897e700726e9a5bf0ecaa"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-64/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "8b31fa3a5f25dc087722a725cf00867b34e1eb7f4bb926b53eb1889ba1e721a9"
"revision": "ops-20261118-r8",
"sha256": "f42e892e33902030d1610fe20cac887e0a597b69cd1ef270e54bb7fc1f41e311"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-65/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "a21a81ff42f8bbaa7abdb8056366d6d3310de870bae256ebdba3eb3b1e9f61fd"
"revision": "ops-20261118-r8",
"sha256": "46a93e25c6ffcc1d9ff9b2e9c6193633ff98fc6babbcbf82fde30155a14997da"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-66/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "5030e944044b5bbbd486d3d110b154438b94d591e7193d9ebcf10f7db08aba28"
"revision": "ops-20261118-r8",
"sha256": "4d16b7badb42781333dc1044c023a2d9d2721f86d065146a87c2764ec2051c1e"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-67/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "c31d1141def7331a39163b2c10d0aa48525edc89feb95c87ec13c5690bdb2ea7"
"revision": "ops-20261118-r8",
"sha256": "3c77cf45729fa8a1c5ae26a304bd30da46b53a5e0dd2e05b7ad0bc9480512844"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-68/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "4e7d8678df929e06fad8ac104e1a4979f42242dec0d9eea754d4367690c5a66f"
"revision": "ops-20261118-r8",
"sha256": "19bc297e09d773fad31e2ee192dc036a9e272735661cbad0da62483b1a7f73da"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-69/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "309ec938ca497309bd79202e6537fd076d13822daaaaad847624368a9b13e61d"
"revision": "ops-20261118-r8",
"sha256": "8880ea501d955ab89236b271df364bf9993fc3f3ca379561e1f7efeb0ece4a84"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-70/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "70fb544cecf6f2230bcd42cdeaf6fb27db38469a2731f509a314598969df883a"
"revision": "ops-20261118-r8",
"sha256": "9180d57147b9bc787227325c6d510dcda9fed3442e72fc8fecf2192377b0ba2b"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-71/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "f360510f6463b823e6a05e072f80dcd393b2e34047236506a7db520f75281b83"
"revision": "ops-20261118-r8",
"sha256": "0a8c8e723487511583d3e59c089223048cc8f68340c743437621ff972ab01d10"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-72/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "63f21cad411257236d8649ef6bec99ae8435d753fe289de745a2dff2330985a4"
"revision": "ops-20261118-r8",
"sha256": "5db004a4152e5c0105319063d06865d378639988392801eaeaa03dc7b0b3d04b"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-73/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "6ccb15b8958e681e2c31baa2d35dd4f6498e74804ca8d40b10eec243c661fb9e"
"revision": "ops-20261118-r8",
"sha256": "ca140b122fd3e80bca6f55ef46efaf33116f91be598bfcceeffff4d375d9d4ef"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-74/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "15139b2c2ff4c13be0056cb087b24f8bbebdb6d7a477eaf5dbe76a7ae33b96fb"
"revision": "ops-20261118-r8",
"sha256": "8f3ac62fdd47d55ef2940202e068548d4800b8ca2e4cd0a260804294b5100a14"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-75/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "c8c4835f7939127550074ba903c9cb58b0fff7f3f54e2c0c0c652a7a6bd4975e"
"revision": "ops-20261118-r8",
"sha256": "b782d5bac6605f1c05cfcded132252119388e667e5559262456a8f00a3d70176"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-76/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "7a122ad0b702adfbd1894f688f5f18f1ea68e56a003043a495cb486c2b972ae8"
"revision": "ops-20261118-r8",
"sha256": "d7c6f88c70f9b43ac5bd5406c2ab82abebc349f8b74616c23e02076c2c8d1895"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-77/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "c69604f7d9b29984929822e1833efd1d438e6312840fa22efd1c6d4ec2ea87cd"
"revision": "ops-20261118-r8",
"sha256": "a697a2edc48aee4bec78b0354233e1c0ecde00dc912e70086e2502b188e9f110"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-78/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "8628fbf34e42638eba8838c8a84c5be6119303a16a021025a59cc37d83227a28"
"revision": "ops-20261118-r8",
"sha256": "a89578e522412060fc20ff236de338208c61cd4c2bdf6f37e9ac5cd307f2027f"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-79/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "b2a6b92351b1dc1bb8d61fb49e33c41f05e9878e11195558773c50c08571be08"
"revision": "ops-20261118-r8",
"sha256": "93a56aed269785ed69c2d3e6f08399084cad66185a6e577c20aced695d3affc8"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-80/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "2d7a8e71ea6fa84cb2a796df97abd5085d346baa8ca69f1060325fb4ea7e36d7"
"revision": "ops-20261118-r8",
"sha256": "b5c9ebddcd79a8c2e74094b65249e330f7fc658b2e27d6d84a263110babc76fa"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-81/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "a87b35ec3bee36db82e20a887da88462365f069a0286089453c1becf93d8b9e6"
"revision": "ops-20261118-r8",
"sha256": "74c12e9b23bf4a4ae30afd0e17ac86df3ea44001e15fd2c793d11148fcc1fffa"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-82/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "58c21f210f82db333b1b786491806d6e38a38fcd04d046ffa84005cb97c124d8"
"revision": "ops-20261118-r8",
"sha256": "f75f2821a09578ca10bffb535f2059c51448b37449f3b9cfc08b7dbb3a41568e"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-83/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "4783b2d6253d6736b438939a5acfa251c6d63b35dc2c63dd11c8965041f93856"
"revision": "ops-20261118-r8",
"sha256": "1e7f7691dea254f1320e67277c5da2f1ea032419ce9428faad46f5b8bbf3419e"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-84/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "39b2cf8f86954bce9279de0ea57a29259055b511232e3354ca4a75944dc952cd"
"revision": "ops-20261118-r8",
"sha256": "e414c52abe5bd1590440a2274248f28da8cdd9d181e7e03bb44eaac8fee6046a"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-85/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "e17100a8e9187fce1036b1c5ff8f704c0fd01a62945b99e5a93022ac79972ef1"
"revision": "ops-20261118-r8",
"sha256": "351c5b1448a41a283766eaa0edc5efd5bfc209753861af82cd3f49fabec82c7b"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-86/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "8e26743289ad6534cebb12d4057241a977d95cdd2f4e7308cca6794f7deff895"
"revision": "ops-20261118-r8",
"sha256": "67e0d5acf149908ad6342e61e80db24fb9497da934ed68b23f49ba39d65d0fac"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions cases/journey-3/J3-87/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
"corpus": "ops-snapshot"
},
"private_spec": {
"revision": "ops-20261118-r5",
"sha256": "ca43f250e17b9de3ff0fd36102fccb7824540d9e401101f24dc5367455217698"
"revision": "ops-20261118-r8",
"sha256": "19e0884866588478e63fb14f86d43988f3a213f3e687c5b4f6eb97057e677638"
},
"sessions": [
{
Expand Down
4 changes: 2 additions & 2 deletions docs/journey-3-operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ Each of the two source agents contributes its 30 slowest distinct questions from

Each replay creates a fresh conversation. Earlier user/assistant text is supplied as quoted background to resolve follow-ups, not replayed as additional agent tasks and not trusted as a reference answer. Attachment transport credentials are omitted. Relative dates use the original question's UTC/Beijing clock, not the workflow date. Queries are read-only. This is a fixed-snapshot question benchmark, not a reproduction of the original model, cache or complete IM session.

The copied agents have no IM/channel bindings. Runtime preflight checks their active status, exact knowledge-base bindings, required skills/tools and the live set of enabled, successfully registered table sources. Importing table declarations alone leaves them legacy-unbound in QA; the supported backfill API registered the copied sources, and their effective active sets now match production (77/64/64/64). The three originally unbound declarations remain disabled. A missing production dependency is `blocked`, not a quality failure for the partially copied agent. The public-cloud replica currently lacks five MCP tools and its dependent custom skill because QA does not discover those exports. It is deliberately not scored as an equivalent production replica.
The copied agents have no IM/channel bindings. Runtime preflight checks their active status, exact knowledge-base bindings, required skills/tools and the live set of enabled, successfully registered table sources. Importing table declarations alone leaves them legacy-unbound in QA; the supported backfill API registered the copied sources, and their effective active sets now match production (77/64/64/64). The three originally unbound declarations remain disabled. A missing production dependency is `blocked`, not a quality failure for the partially copied agent. The public-cloud replica binds five MCP tools and its custom skill to a Journey-3-only adapter for the QA semantic API. Read, list and validate target the copied snapshot. Create and update target an unbound scratch model. Case revision r8 records the QA bindings while preserving the reference questions and answers. See [the adapter](../tools/journey3-semantic-mcp/README.md).

Seven original requests depend on monitoring, GitHub, image delivery, old-task introspection or customer business tables outside the copied knowledge bases. They remain in the selected 60 as `out_of_scope`, with no knowledge-quality score. The original 91-row uploaded list for a further identity-mapping question was recovered and checksum-verified. Its verbatim input and independently calculated account mappings stay in QA; duplicate emails and ambiguous account links remain explicit. These exclusions remain visible rather than being replaced with easier questions.

Expand Down Expand Up @@ -45,4 +45,4 @@ Every Journey-3 case uses the same publication boundary, including the old synth

## Validation status

Selection, source export, independent reference calculations, QA metadata/spec registration and framework implementation are prepared. All 272 copied tables passed final QA row-count verification (35,470,556 rows), including the independently sealed 192-table private-cloud subset. Fifty-three selected questions now have usable independent knowledge-answer references; seven are outside the copied knowledge bases. A real private-cloud agent request reached a failed terminal state after about 82 seconds with `insufficient_credit`; its trace, session and 29 events were saved to QA, and the framework completed successfully. No successful new model-based evaluation is claimed. The incomplete MCP replica and external-scope questions remain explicitly diagnosed. No full QA suite or scheduled run is enabled by this change.
Selection, source export, independent reference calculations, QA metadata/spec registration and framework implementation are prepared. All 272 copied tables passed final QA row-count verification (35,470,556 rows), including the independently sealed 192-table private-cloud subset. Fifty-three selected questions now have usable independent knowledge-answer references; seven are outside the copied knowledge bases. A real private-cloud agent request reached a failed terminal state after about 82 seconds with `insufficient_credit`; its trace, session and 29 events were saved to QA, and the framework completed successfully. No successful new model-based evaluation is claimed. The MCP adapter and bindings are now verified. The four public-cloud and three private-cloud questions requiring data beyond the copied knowledge bases remain explicitly out of scope. No full QA suite or scheduled run is enabled by this change.
49 changes: 49 additions & 0 deletions tests/test_semantic_mcp_bridge.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import importlib.util
from pathlib import Path
import unittest
from unittest.mock import Mock


path = Path(__file__).resolve().parents[1] / 'tools/journey3-semantic-mcp/bridge.py'
spec = importlib.util.spec_from_file_location('semantic_bridge', path)
bridge = importlib.util.module_from_spec(spec)
spec.loader.exec_module(bridge)

READ = 'newmoi_get_semantic_models_model_id_entries'
CREATE = 'newmoi_post_semantic_models_model_id_entries'
UPDATE = 'newmoi_put_semantic_models_model_id_entries_entry_id'


class SemanticMCPBridgeTests(unittest.TestCase):
def setUp(self):
self.upstream = Mock(return_value={'ok': True})
self.service = bridge.Bridge('journey-3', {80001}, 80005, [], self.upstream)

def call(self, name, model, **extra):
return self.service.call({'name': name, 'arguments': {
'workspace_id': 'journey-3', 'model_id': model, **extra,
}})

def test_uses_real_rest_contract_and_projects_data_for_mcp(self):
result = self.call(READ, 80001, page_size=1, domain='test & literal')
self.assertEqual(result['structuredContent'], {'ok': True})
self.upstream.assert_called_once_with('GET',
'/semantic-models/80001/entries?domain=test+%26+literal&page_size=1', None)

def test_rejects_cross_workspace_and_all_writes_to_fixed_snapshot(self):
for name, arguments in [
(READ, {'workspace_id': 'production'}),
(READ, {'model_id': 90002}),
(CREATE, {'kind': 'business_rule', 'key': 'fixture', 'spec': {}}),
(UPDATE, {'entry_id': 5, 'kind': 'business_rule', 'key': 'fixture', 'spec': {}}),
]:
with self.subTest(name=name, arguments=arguments):
values = {'workspace_id': 'journey-3', 'model_id': 80001, **arguments}
self.assertTrue(self.service.call({'name': name, 'arguments': values})['isError'])
self.upstream.assert_not_called()

def test_updates_only_scratch_and_treats_id_as_integer(self):
data = {'kind': 'business_rule', 'key': 'fixture', 'spec': {'summary': 'fixture'}}
self.assertTrue(self.call(UPDATE, 80005, entry_id='../80001', **data)['isError'])
self.assertEqual(self.call(UPDATE, 80005, entry_id=123, **data)['structuredContent'], {'ok': True})
self.upstream.assert_called_once_with('PUT', '/semantic-models/80005/entries/123', data)
Loading
Loading