diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/GetDomainsTool.java b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/GetDomainsTool.java index c53fcf0..792c359 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/GetDomainsTool.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/GetDomainsTool.java @@ -18,7 +18,7 @@ package io.github.malonetalk.agent.tools; import io.agentscope.core.tool.Tool; -import io.github.malonetalk.service.DomainService; +import io.github.malonetalk.service.semantic.DomainService; import java.util.List; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/controller/DomainController.java b/data-agent-backend/src/main/java/io/github/malonetalk/controller/DomainController.java index 3f905c1..f228756 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/controller/DomainController.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/controller/DomainController.java @@ -23,7 +23,7 @@ import io.github.malonetalk.dto.DomainUpdateRequest; import io.github.malonetalk.dto.pagination.PageResponse; import io.github.malonetalk.entity.DomainInfo; -import io.github.malonetalk.service.DomainService; +import io.github.malonetalk.service.semantic.DomainService; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; import org.springframework.web.bind.annotation.DeleteMapping; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableRelationWorkspaceController.java b/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableRelationWorkspaceController.java new file mode 100644 index 0000000..c2827a7 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableRelationWorkspaceController.java @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.controller; + +import io.github.malonetalk.common.Result; +import io.github.malonetalk.dto.semantic.RelationWorkspacePageQuery; +import io.github.malonetalk.dto.semantic.RelationWorkspaceResponse; +import io.github.malonetalk.service.semantic.relation.RelationSemanticService; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/semantic/tables/relations/workspace") +@RequiredArgsConstructor +public class TableRelationWorkspaceController { + + private final RelationSemanticService relationSemanticService; + + @GetMapping + public Result getWorkspace(@Valid RelationWorkspacePageQuery query) { + return Result.success(relationSemanticService.getRelationWorkspace(query)); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableSemanticSyncController.java b/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableSemanticSyncController.java new file mode 100644 index 0000000..e155adb --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableSemanticSyncController.java @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.controller; + +import io.github.malonetalk.common.Result; +import io.github.malonetalk.dto.pagination.PageResponse; +import io.github.malonetalk.dto.semantic.PhysicalTableCandidatePageQuery; +import io.github.malonetalk.dto.semantic.PhysicalTableCandidateResponse; +import io.github.malonetalk.dto.semantic.RefreshPhysicalStatusRequest; +import io.github.malonetalk.dto.semantic.SyncTableSemanticsRequest; +import io.github.malonetalk.dto.semantic.SyncTableSemanticsResponse; +import io.github.malonetalk.service.semantic.sync.SemanticSyncService; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/semantic/tables/sync") +@RequiredArgsConstructor +public class TableSemanticSyncController { + + private final SemanticSyncService semanticSyncService; + + @GetMapping("/candidates") + public Result> getPhysicalTableCandidates( + @Valid PhysicalTableCandidatePageQuery query) { + return Result.success(semanticSyncService.getPhysicalTableCandidates(query)); + } + + @PostMapping + public Result syncTables( + @Valid @RequestBody SyncTableSemanticsRequest request) { + return Result.success(semanticSyncService.syncTables(request)); + } + + @PostMapping("/physical-status") + public Result refreshPhysicalStatus( + @Valid @RequestBody RefreshPhysicalStatusRequest request) { + return Result.success(semanticSyncService.refreshPhysicalStatus(request)); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/convertor/PromptConverter.java b/data-agent-backend/src/main/java/io/github/malonetalk/convertor/PromptConverter.java index ec340e0..7874873 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/convertor/PromptConverter.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/convertor/PromptConverter.java @@ -25,11 +25,10 @@ import io.github.malonetalk.dto.prompt.TableRelationPromptResponse; import io.github.malonetalk.entity.ColumnInfo; import io.github.malonetalk.entity.TableInfo; +import io.github.malonetalk.service.semantic.SemanticAvailabilityHelper; import io.github.malonetalk.utils.SemanticUtils; import java.util.List; -import java.util.Locale; import java.util.Map; -import java.util.Optional; /** 物理层/语义层 → Agent Prompt DTO 的统一转换器,集中管理所有面向 LLM 的 DTO 映射逻辑。 */ public final class PromptConverter { @@ -40,16 +39,16 @@ private PromptConverter() {} public static ColumnPromptResponse mapColumnPrompt( PhysicalColumnInfo physicalColumn, Map semanticByName) { ColumnInfo semanticColumn = - semanticByName.get(physicalColumn.columnName().toLowerCase(Locale.ROOT)); - if (semanticColumn != null && !Boolean.TRUE.equals(semanticColumn.getIsVisible())) { + semanticByName.get(SemanticUtils.objectKey(physicalColumn.columnName())); + if (!SemanticAvailabilityHelper.isColumnAvailable( + semanticColumn, SemanticAvailabilityHelper.UsageLevel.AI_PROMPT)) { return null; } String description = - Optional.ofNullable(semanticColumn) - .map(ColumnInfo::getColumnDescription) - .map(SemanticUtils::trimToNull) - .orElseGet(() -> SemanticUtils.trimToNull(physicalColumn.remarks())); + SemanticUtils.firstNonBlank( + semanticColumn == null ? null : semanticColumn.getColumnDescription(), + physicalColumn.remarks()); StringBuilder typeBuilder = new StringBuilder(physicalColumn.typeName()); if (physicalColumn.columnSize() > 0) { @@ -72,8 +71,9 @@ public static TablePromptResponse mapTablePrompt( Map semanticByName, List resolvedRelations) { TableInfo semanticTable = - semanticByName.get(physicalTable.tableName().toLowerCase(Locale.ROOT)); - if (semanticTable != null && !Boolean.TRUE.equals(semanticTable.getIsVisible())) { + semanticByName.get(SemanticUtils.objectKey(physicalTable.tableName())); + if (!SemanticAvailabilityHelper.isTableAvailable( + semanticTable, SemanticAvailabilityHelper.UsageLevel.AI_PROMPT)) { return null; } @@ -93,9 +93,8 @@ private static String resolveDomain(TableInfo semanticTable) { private static String resolveDescription( PhysicalTableInfo physicalTable, TableInfo semanticTable) { - return Optional.ofNullable(semanticTable) - .map(TableInfo::getTableDescription) - .map(SemanticUtils::trimToNull) - .orElseGet(() -> SemanticUtils.trimToNull(physicalTable.remarks())); + return SemanticUtils.firstNonBlank( + semanticTable == null ? null : semanticTable.getTableDescription(), + physicalTable.remarks()); } } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/convertor/SemanticConverter.java b/data-agent-backend/src/main/java/io/github/malonetalk/convertor/SemanticConverter.java index d2192d6..5b2e568 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/convertor/SemanticConverter.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/convertor/SemanticConverter.java @@ -20,11 +20,14 @@ import io.github.malonetalk.common.SemanticConstants; import io.github.malonetalk.dto.semantic.ColumnSemanticResponse; import io.github.malonetalk.dto.semantic.LogicalTableRelationResponse; +import io.github.malonetalk.dto.semantic.RelationWorkspaceColumnResponse; +import io.github.malonetalk.dto.semantic.RelationWorkspaceTableResponse; import io.github.malonetalk.dto.semantic.TableSemanticResponse; import io.github.malonetalk.entity.ColumnInfo; import io.github.malonetalk.entity.LogicalTableRelation; import io.github.malonetalk.entity.TableInfo; import io.github.malonetalk.enums.LogicalTableRelationType; +import io.github.malonetalk.service.semantic.SemanticAvailabilityHelper; import io.github.malonetalk.service.semantic.relation.LogicalTableRelationHelper; import io.github.malonetalk.utils.SemanticUtils; import java.util.List; @@ -39,25 +42,41 @@ public class SemanticConverter { public TableSemanticResponse toResponse(TableInfo tableInfo) { Boolean isVisible = tableInfo.getIsVisible(); + boolean hasPhysicalTable = SemanticAvailabilityHelper.hasPhysicalTable(tableInfo); return TableSemanticResponse.builder() .id(tableInfo.getId()) .tableName(tableInfo.getTableName()) .domain(SemanticUtils.normalizeDomain(tableInfo.getDomain())) + .physicalTableDescription( + SemanticUtils.trimToNull(tableInfo.getPhysicalTableDescription())) .tableDescription(SemanticUtils.trimToNull(tableInfo.getTableDescription())) .isVisible(isVisible) - .hasPhysicalTable(true) + .hasPhysicalTable(hasPhysicalTable) + .invalidReason( + SemanticAvailabilityHelper.tableInvalidReason( + tableInfo, SemanticAvailabilityHelper.UsageLevel.USER_OPERATION)) .updateTime(tableInfo.getUpdateTime()) .build(); } public ColumnSemanticResponse toResponse(ColumnInfo columnInfo) { + boolean hasPhysicalColumn = SemanticAvailabilityHelper.hasPhysicalColumn(columnInfo); return ColumnSemanticResponse.builder() .id(columnInfo.getId()) .columnName(columnInfo.getColumnName()) + .physicalColumnDescription( + SemanticUtils.trimToNull(columnInfo.getPhysicalColumnDescription())) .columnDescription(columnInfo.getColumnDescription()) + .typeName(SemanticUtils.trimToNull(columnInfo.getTypeName())) + .primaryKey(columnInfo.getPrimaryKey()) .isVisible(columnInfo.getIsVisible()) - .hasPhysicalColumn(true) - .effective(Boolean.TRUE.equals(columnInfo.getIsVisible())) + .hasPhysicalColumn(hasPhysicalColumn) + .effective( + SemanticAvailabilityHelper.isColumnAvailable( + columnInfo, SemanticAvailabilityHelper.UsageLevel.USER_OPERATION)) + .invalidReason( + SemanticAvailabilityHelper.columnInvalidReason( + columnInfo, SemanticAvailabilityHelper.UsageLevel.USER_OPERATION)) .updateTime(columnInfo.getUpdateTime()) .build(); } @@ -91,4 +110,32 @@ public LogicalTableRelationResponse toResponse(LogicalTableRelation relation) { .updateTime(relation.getUpdateTime()) .build(); } + + public RelationWorkspaceTableResponse toWorkspaceTable( + TableInfo tableInfo, List columns) { + return new RelationWorkspaceTableResponse( + tableInfo.getTableName(), + SemanticUtils.normalizeDomain(tableInfo.getDomain()), + SemanticUtils.firstNonBlank( + tableInfo.getTableDescription(), tableInfo.getPhysicalTableDescription()), + SemanticAvailabilityHelper.isTableAvailable( + tableInfo, SemanticAvailabilityHelper.UsageLevel.USER_OPERATION), + SemanticAvailabilityHelper.tableInvalidReason( + tableInfo, SemanticAvailabilityHelper.UsageLevel.USER_OPERATION), + columns.stream().map(this::toWorkspaceColumn).toList()); + } + + public RelationWorkspaceColumnResponse toWorkspaceColumn(ColumnInfo columnInfo) { + return new RelationWorkspaceColumnResponse( + columnInfo.getColumnName(), + SemanticUtils.firstNonBlank( + columnInfo.getColumnDescription(), + columnInfo.getPhysicalColumnDescription()), + SemanticUtils.trimToNull(columnInfo.getTypeName()), + columnInfo.getPrimaryKey(), + SemanticAvailabilityHelper.isColumnAvailable( + columnInfo, SemanticAvailabilityHelper.UsageLevel.USER_OPERATION), + SemanticAvailabilityHelper.columnInvalidReason( + columnInfo, SemanticAvailabilityHelper.UsageLevel.USER_OPERATION)); + } } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/PhysicalTableCandidatePageQuery.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/PhysicalTableCandidatePageQuery.java new file mode 100644 index 0000000..a6ca465 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/PhysicalTableCandidatePageQuery.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; + +public record PhysicalTableCandidatePageQuery( + @NotNull(message = "datasourceId 不能为空") Integer datasourceId, + @Min(value = 1, message = "page 不能小于 1") Integer page, + @Min(value = 1, message = "pageSize 不能小于 1") Integer pageSize, + String keyword, + String sortOrder) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/PhysicalTableCandidateResponse.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/PhysicalTableCandidateResponse.java new file mode 100644 index 0000000..9c1acb0 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/PhysicalTableCandidateResponse.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +public record PhysicalTableCandidateResponse( + String tableName, String physicalTableDescription, Boolean synced) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RefreshPhysicalStatusRequest.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RefreshPhysicalStatusRequest.java new file mode 100644 index 0000000..65627be --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RefreshPhysicalStatusRequest.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +import jakarta.validation.constraints.NotNull; + +public record RefreshPhysicalStatusRequest( + @NotNull(message = "datasourceId 不能为空") Integer datasourceId) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceColumnResponse.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceColumnResponse.java new file mode 100644 index 0000000..83b9b14 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceColumnResponse.java @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +public record RelationWorkspaceColumnResponse( + String columnName, + String description, + String typeName, + Boolean primaryKey, + boolean operable, + String invalidReason) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspacePageQuery.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspacePageQuery.java new file mode 100644 index 0000000..f4a8fa0 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspacePageQuery.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Pattern; + +public record RelationWorkspacePageQuery( + @NotNull @Min(1) Integer datasourceId, + @Min(1) Integer page, + @Min(1) Integer pageSize, + String keyword, + @Pattern(regexp = "^(?i)(asc|desc)$", message = "sortOrder must be asc or desc.") + String sortOrder) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceResponse.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceResponse.java new file mode 100644 index 0000000..071c6fc --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceResponse.java @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +import io.github.malonetalk.dto.pagination.PageResponse; +import java.util.List; + +public record RelationWorkspaceResponse( + PageResponse nodes, + List relations) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceTableResponse.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceTableResponse.java new file mode 100644 index 0000000..18b1b04 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationWorkspaceTableResponse.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +import java.util.List; + +public record RelationWorkspaceTableResponse( + String tableName, + String domain, + String description, + boolean operable, + String invalidReason, + List columns) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableResult.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableResult.java new file mode 100644 index 0000000..a1d1860 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableResult.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +public record SyncTableResult( + String tableName, + boolean physicalTableFound, + boolean tableAdded, + boolean tableReactivated, + boolean tableUpdated, + boolean tableMarkedMissing, + int addedColumns, + int reactivatedColumns, + int updatedColumns, + int missingColumnsMarked, + String message) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableSemanticsRequest.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableSemanticsRequest.java new file mode 100644 index 0000000..5098f01 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableSemanticsRequest.java @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import java.util.List; + +public record SyncTableSemanticsRequest( + @NotNull(message = "datasourceId 不能为空") Integer datasourceId, + @NotEmpty(message = "tableNames 不能为空") List tableNames) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableSemanticsResponse.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableSemanticsResponse.java new file mode 100644 index 0000000..deff25e --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/SyncTableSemanticsResponse.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +import java.util.List; + +public record SyncTableSemanticsResponse( + int addedTables, + int reactivatedTables, + int updatedTables, + int missingTablesMarked, + int addedColumns, + int reactivatedColumns, + int updatedColumns, + int missingColumnsMarked, + List results) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/entity/ColumnInfo.java b/data-agent-backend/src/main/java/io/github/malonetalk/entity/ColumnInfo.java index 45bb532..e3df009 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/entity/ColumnInfo.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/entity/ColumnInfo.java @@ -27,8 +27,12 @@ public class ColumnInfo { private Integer datasourceId; private String tableName; private String columnName; + private String physicalColumnDescription; + private String typeName; + private Boolean primaryKey; private String columnDescription; private Boolean isVisible; + private Boolean physicalStatus; private LocalDateTime createTime; private LocalDateTime updateTime; } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/entity/TableInfo.java b/data-agent-backend/src/main/java/io/github/malonetalk/entity/TableInfo.java index cb2c16f..653e429 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/entity/TableInfo.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/entity/TableInfo.java @@ -25,10 +25,12 @@ public class TableInfo { private Integer id; private String tableName; + private String physicalTableDescription; private String tableDescription; private String domain; private Integer datasourceId; private Boolean isVisible; + private Boolean physicalStatus; private LocalDateTime createTime; private LocalDateTime updateTime; } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticInfoMapper.java b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticInfoMapper.java index 470e761..c8454fe 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticInfoMapper.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticInfoMapper.java @@ -31,6 +31,10 @@ public interface ColumnSemanticInfoMapper { List selectByDatasourceIdAndTableName( @Param("datasourceId") Integer datasourceId, @Param("tableName") String tableName); + List selectByDatasourceIdAndTableNames( + @Param("datasourceId") Integer datasourceId, + @Param("tableNames") List tableNames); + List selectPageByDatasourceIdAndTableName( @Param("query") ColumnSemanticPageQuery query, @Param("sortDescending") boolean sortDescending); @@ -42,7 +46,9 @@ ColumnInfo selectByDatasourceIdAndTableNameAndColumnName( int insert(ColumnInfo columnInfo); - int update(ColumnInfo columnInfo); + int updateSemanticFields(ColumnInfo columnInfo); + + int updatePhysicalCacheFields(ColumnInfo columnInfo); int deleteByDatasourceId(@Param("datasourceId") Integer datasourceId); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/LogicalTableRelationMapper.java b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/LogicalTableRelationMapper.java index f68b5d2..e915ad3 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/LogicalTableRelationMapper.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/LogicalTableRelationMapper.java @@ -35,6 +35,10 @@ List selectByDatasourceIdAndSourceTable( @Param("datasourceId") Integer datasourceId, @Param("sourceTableName") String sourceTableName); + List selectByDatasourceIdAndSourceTables( + @Param("datasourceId") Integer datasourceId, + @Param("sourceTableNames") List sourceTableNames); + List selectPageByDatasourceIdAndSourceTable( @Param("query") RelationSemanticPageQuery query, @Param("sortDescending") boolean sortDescending); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableInfoMapper.java b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableInfoMapper.java index 92bfee3..9381c62 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableInfoMapper.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableInfoMapper.java @@ -28,7 +28,9 @@ public interface TableInfoMapper { int insert(TableInfo tableInfo); - int update(TableInfo tableInfo); + int updateSemanticFields(TableInfo tableInfo); + + int updatePhysicalCacheFields(TableInfo tableInfo); int deleteByDatasourceIdAndIds( @Param("datasourceId") Integer datasourceId, @Param("ids") List ids); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/DomainService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/DomainService.java similarity index 96% rename from data-agent-backend/src/main/java/io/github/malonetalk/service/DomainService.java rename to data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/DomainService.java index cfb6117..fdfd11f 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/DomainService.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/DomainService.java @@ -15,7 +15,7 @@ * along with this program. If not, see . * limitations under the License. */ -package io.github.malonetalk.service; +package io.github.malonetalk.service.semantic; import io.github.malonetalk.dto.DomainCreateRequest; import io.github.malonetalk.dto.DomainPageQuery; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/DomainServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/DomainServiceImpl.java similarity index 94% rename from data-agent-backend/src/main/java/io/github/malonetalk/service/DomainServiceImpl.java rename to data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/DomainServiceImpl.java index 59579e2..9e4bef1 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/DomainServiceImpl.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/DomainServiceImpl.java @@ -15,7 +15,7 @@ * along with this program. If not, see . * limitations under the License. */ -package io.github.malonetalk.service; +package io.github.malonetalk.service.semantic; import com.github.pagehelper.Page; import com.github.pagehelper.PageHelper; @@ -30,7 +30,6 @@ import io.github.malonetalk.utils.SemanticUtils; import java.time.LocalDateTime; import java.util.List; -import java.util.Locale; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -72,8 +71,7 @@ public DomainInfo findById(Integer id) { @Override public DomainInfo create(DomainCreateRequest request) { - String normalizedName = - SemanticUtils.trimToNotBlank(request.name(), "领域名称").toLowerCase(Locale.ROOT); + String normalizedName = SemanticUtils.objectKey(request.name(), "领域名称"); DomainInfo existing = domainInfoMapper.selectByName(normalizedName); if (existing != null) { throw new IllegalArgumentException("领域名称已存在: " + normalizedName); @@ -96,8 +94,7 @@ public DomainInfo update(Integer id, DomainUpdateRequest request) { if (existing == null) { throw new IllegalArgumentException("领域不存在: id=" + id); } - String normalizedName = - SemanticUtils.trimToNotBlank(request.name(), "领域名称").toLowerCase(Locale.ROOT); + String normalizedName = SemanticUtils.objectKey(request.name(), "领域名称"); DomainInfo nameConflict = domainInfoMapper.selectByName(normalizedName); if (nameConflict != null && !nameConflict.getId().equals(id)) { throw new IllegalArgumentException("领域名称已存在: " + normalizedName); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticAvailabilityHelper.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticAvailabilityHelper.java new file mode 100644 index 0000000..2945fd7 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticAvailabilityHelper.java @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.semantic; + +import io.github.malonetalk.entity.ColumnInfo; +import io.github.malonetalk.entity.TableInfo; + +public final class SemanticAvailabilityHelper { + + private SemanticAvailabilityHelper() {} + + public enum UsageLevel { + AI_PROMPT, + FRONTEND_DISPLAY, + USER_OPERATION + } + + public static boolean isTableAvailable(TableInfo tableInfo, UsageLevel usageLevel) { + if (tableInfo == null) { + return true; + } + if (usageLevel == UsageLevel.FRONTEND_DISPLAY) { + return true; + } + return Boolean.TRUE.equals(tableInfo.getIsVisible()) && hasPhysicalTable(tableInfo); + } + + public static boolean isColumnAvailable(ColumnInfo columnInfo, UsageLevel usageLevel) { + if (columnInfo == null) { + return true; + } + if (usageLevel == UsageLevel.FRONTEND_DISPLAY) { + return true; + } + return Boolean.TRUE.equals(columnInfo.getIsVisible()) && hasPhysicalColumn(columnInfo); + } + + public static boolean hasPhysicalTable(TableInfo tableInfo) { + return tableInfo == null || !Boolean.FALSE.equals(tableInfo.getPhysicalStatus()); + } + + public static boolean hasPhysicalColumn(ColumnInfo columnInfo) { + return columnInfo == null || !Boolean.FALSE.equals(columnInfo.getPhysicalStatus()); + } + + public static boolean isUnavailable(TableInfo tableInfo, UsageLevel usageLevel) { + return !isTableAvailable(tableInfo, usageLevel); + } + + public static boolean isUnavailable(ColumnInfo columnInfo, UsageLevel usageLevel) { + return !isColumnAvailable(columnInfo, usageLevel); + } + + public static String tableInvalidReason(TableInfo tableInfo, UsageLevel usageLevel) { + if (isTableAvailable(tableInfo, usageLevel)) { + return null; + } + if (!hasPhysicalTable(tableInfo)) { + return "物理表不存在"; + } + if (!Boolean.TRUE.equals(tableInfo.getIsVisible())) { + return "表已隐藏"; + } + return "表不可用"; + } + + public static String columnInvalidReason(ColumnInfo columnInfo, UsageLevel usageLevel) { + if (isColumnAvailable(columnInfo, usageLevel)) { + return null; + } + if (!hasPhysicalColumn(columnInfo)) { + return "物理列不存在"; + } + if (!Boolean.TRUE.equals(columnInfo.getIsVisible())) { + return "列已隐藏"; + } + return "列不可用"; + } + + public static String unavailableMessage(String fieldName, String objectName, String reason) { + String fallbackReason = reason == null || reason.isBlank() ? "不可用" : reason; + return fieldName + " " + objectName + " is unavailable: " + fallbackReason; + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticMergeService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticMergeService.java index ae9b8d3..a8b9010 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticMergeService.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticMergeService.java @@ -39,7 +39,6 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -97,6 +96,10 @@ public List getTableSchema(Datasource datasource, String t TableInfo semanticTable = tableInfoMapper.selectByDatasourceIdAndTableName( datasource.getId(), normalizedTableName); + if (semanticTable != null && !SemanticAvailabilityHelper.hasPhysicalTable(semanticTable)) { + throw new IllegalArgumentException( + "Table " + normalizedTableName + " does not exist physically."); + } if (semanticTable != null && !Boolean.TRUE.equals(semanticTable.getIsVisible())) { throw new IllegalArgumentException("Table " + normalizedTableName + " is hidden."); } @@ -209,7 +212,7 @@ private Map buildSemanticColumnIndex( for (ColumnInfo column : columnSemanticInfoMapper.selectByDatasourceIdAndTableName( datasourceId, tableName)) { - result.put(column.getColumnName().toLowerCase(Locale.ROOT), column); + result.put(SemanticUtils.objectKey(column.getColumnName()), column); } return result; } @@ -249,7 +252,7 @@ private record TableNameIndex(Map index) { private static TableNameIndex of(List tables) { Map map = new HashMap<>(); for (TableInfo table : tables) { - map.put(table.getTableName().toLowerCase(Locale.ROOT), table); + map.put(SemanticUtils.objectKey(table.getTableName()), table); } return new TableNameIndex(map); } @@ -259,12 +262,13 @@ private Map asMap() { } private TableInfo get(String tableName) { - return index.get(tableName.toLowerCase(Locale.ROOT)); + return index.get(SemanticUtils.objectKey(tableName)); } private boolean isHidden(String tableName) { TableInfo tableInfo = get(tableName); - return tableInfo != null && !Boolean.TRUE.equals(tableInfo.getIsVisible()); + return !SemanticAvailabilityHelper.isTableAvailable( + tableInfo, SemanticAvailabilityHelper.UsageLevel.AI_PROMPT); } } @@ -274,22 +278,23 @@ private static TableColumnIndex of(List columns) { Map> map = new HashMap<>(); for (ColumnInfo column : columns) { map.computeIfAbsent( - column.getTableName().toLowerCase(Locale.ROOT), + SemanticUtils.objectKey(column.getTableName()), key -> new HashMap<>()) - .put(column.getColumnName().toLowerCase(Locale.ROOT), column); + .put(SemanticUtils.objectKey(column.getColumnName()), column); } return new TableColumnIndex(map); } private ColumnInfo get(String tableName, String columnName) { - Map columns = index.get(tableName.toLowerCase(Locale.ROOT)); - return columns == null ? null : columns.get(columnName.toLowerCase(Locale.ROOT)); + Map columns = index.get(SemanticUtils.objectKey(tableName)); + return columns == null ? null : columns.get(SemanticUtils.objectKey(columnName)); } private boolean hasHiddenColumn(String tableName, List columnNames) { for (String columnName : columnNames) { ColumnInfo columnInfo = get(tableName, columnName); - if (columnInfo != null && !Boolean.TRUE.equals(columnInfo.getIsVisible())) { + if (!SemanticAvailabilityHelper.isColumnAvailable( + columnInfo, SemanticAvailabilityHelper.UsageLevel.AI_PROMPT)) { return true; } } @@ -303,7 +308,7 @@ private static RelationSourceIndex of(List relations) { Map> map = new HashMap<>(); for (LogicalTableRelation relation : relations) { map.computeIfAbsent( - relation.getSourceTableName().toLowerCase(Locale.ROOT), + SemanticUtils.objectKey(relation.getSourceTableName()), key -> new ArrayList<>()) .add(relation); } @@ -312,7 +317,7 @@ private static RelationSourceIndex of(List relations) { private List get(String sourceTableName) { return index.getOrDefault( - sourceTableName.toLowerCase(Locale.ROOT), Collections.emptyList()); + SemanticUtils.objectKey(sourceTableName), Collections.emptyList()); } } } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/column/ColumnSemanticServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/column/ColumnSemanticServiceImpl.java index 87e723e..8964e6d 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/column/ColumnSemanticServiceImpl.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/column/ColumnSemanticServiceImpl.java @@ -33,7 +33,6 @@ import io.github.malonetalk.utils.SemanticUtils; import java.time.LocalDateTime; import java.util.List; -import java.util.Locale; import java.util.Set; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; @@ -79,11 +78,8 @@ public PageResponse getColumnPage(ColumnSemanticPageQuer @Override public void updateColumnSemantic(String tableName, ColumnSemanticUpdateRequest request) { requireDatasource(request.datasourceId()); - String normalizedTableName = - SemanticUtils.trimToNotBlank(tableName, "tableName").toLowerCase(Locale.ROOT); - String normalizedColumnName = - SemanticUtils.trimToNotBlank(request.columnName(), "columnName") - .toLowerCase(Locale.ROOT); + String normalizedTableName = SemanticUtils.objectKey(tableName, "tableName"); + String normalizedColumnName = SemanticUtils.objectKey(request.columnName(), "columnName"); ColumnInfo existing = columnSemanticInfoMapper.selectByDatasourceIdAndTableNameAndColumnName( request.datasourceId(), normalizedTableName, normalizedColumnName); @@ -94,6 +90,7 @@ public void updateColumnSemantic(String tableName, ColumnSemanticUpdateRequest r columnInfo.setColumnName(normalizedColumnName); columnInfo.setColumnDescription(SemanticUtils.trimToNull(request.columnDescription())); columnInfo.setIsVisible(request.isVisible()); + columnInfo.setPhysicalStatus(Boolean.FALSE); columnInfo.setCreateTime(LocalDateTime.now()); columnInfo.setUpdateTime(LocalDateTime.now()); columnSemanticInfoMapper.insert(columnInfo); @@ -104,7 +101,7 @@ public void updateColumnSemantic(String tableName, ColumnSemanticUpdateRequest r existing.setColumnDescription(SemanticUtils.trimToNull(request.columnDescription())); existing.setIsVisible(request.isVisible()); existing.setUpdateTime(LocalDateTime.now()); - columnSemanticInfoMapper.update(existing); + columnSemanticInfoMapper.updateSemanticFields(existing); } @Override @@ -137,10 +134,7 @@ public int resetColumnSemantics( } Set normalizedColumnNames = columnNames.stream() - .map( - columnName -> - SemanticUtils.trimToNotBlank(columnName, "columnName") - .toLowerCase(Locale.ROOT)) + .map(columnName -> SemanticUtils.objectKey(columnName, "columnName")) .collect(Collectors.toCollection(java.util.LinkedHashSet::new)); List matchedIds = columnSemanticInfoMapper @@ -149,10 +143,7 @@ public int resetColumnSemantics( .filter( column -> normalizedColumnNames.contains( - SemanticUtils.trimToNotBlank( - column.getColumnName(), - "columnName") - .toLowerCase(Locale.ROOT))) + SemanticUtils.objectKey(column.getColumnName()))) .map(ColumnInfo::getId) .distinct() .toList(); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/LogicalTableRelationHelper.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/LogicalTableRelationHelper.java index edd9d45..1f05c1a 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/LogicalTableRelationHelper.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/LogicalTableRelationHelper.java @@ -27,7 +27,6 @@ import io.github.malonetalk.utils.SemanticUtils; import java.util.LinkedHashSet; import java.util.List; -import java.util.Locale; import java.util.Set; import org.springframework.stereotype.Component; @@ -43,7 +42,7 @@ public LogicalTableRelationHelper(ObjectMapper objectMapper) { } public String normalizeTableName(String tableName, String label) { - return SemanticUtils.trimToNotBlank(tableName, label).toLowerCase(Locale.ROOT); + return SemanticUtils.objectKey(tableName, label); } public List normalizeColumnNames(List columnNames, String fieldName) { @@ -57,7 +56,7 @@ public List normalizeColumnNames(List columnNames, String fieldN throw new IllegalArgumentException(fieldName + " contains a blank column name."); } String normalizedColumnName = columnName.trim(); - String uniqueKey = normalizedColumnName.toLowerCase(Locale.ROOT); + String uniqueKey = SemanticUtils.objectKey(normalizedColumnName); if (!uniqueKeys.add(uniqueKey)) { throw new IllegalArgumentException( fieldName + " contains duplicate column: " + normalizedColumnName); @@ -69,7 +68,7 @@ public List normalizeColumnNames(List columnNames, String fieldN public String buildColumnSignature(List columnNames) { return normalizeColumnNames(columnNames, "columnNames").stream() - .map(columnName -> columnName.toLowerCase(Locale.ROOT)) + .map(SemanticUtils::objectKey) .reduce((left, right) -> left + RELATION_KEY_SEPARATOR + right) .orElse(""); } @@ -79,13 +78,11 @@ public String buildRelationKey( List sourceColumnNames, String targetTableName, List targetColumnNames) { - return SemanticUtils.trimToNotBlank(sourceTableName, "sourceTableName") - .toLowerCase(Locale.ROOT) + return SemanticUtils.objectKey(sourceTableName, "sourceTableName") + RELATION_TABLE_COLUMN_SEPARATOR + buildColumnSignature(sourceColumnNames) + RELATION_GROUP_SEPARATOR - + SemanticUtils.trimToNotBlank(targetTableName, "targetTableName") - .toLowerCase(Locale.ROOT) + + SemanticUtils.objectKey(targetTableName, "targetTableName") + RELATION_TABLE_COLUMN_SEPARATOR + buildColumnSignature(targetColumnNames); } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticService.java index bafecff..676b74c 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticService.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticService.java @@ -21,6 +21,8 @@ import io.github.malonetalk.dto.semantic.BindLogicalTableRelationRequest; import io.github.malonetalk.dto.semantic.LogicalTableRelationResponse; import io.github.malonetalk.dto.semantic.RelationSemanticPageQuery; +import io.github.malonetalk.dto.semantic.RelationWorkspacePageQuery; +import io.github.malonetalk.dto.semantic.RelationWorkspaceResponse; import io.github.malonetalk.dto.semantic.UpdateLogicalTableRelationEnabledRequest; import io.github.malonetalk.dto.semantic.UpdateLogicalTableRelationRequest; import java.util.List; @@ -29,6 +31,8 @@ public interface RelationSemanticService { PageResponse getRelationPage(RelationSemanticPageQuery query); + RelationWorkspaceResponse getRelationWorkspace(RelationWorkspacePageQuery query); + LogicalTableRelationResponse createRelationSemantic( String tableName, BindLogicalTableRelationRequest request); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticServiceImpl.java index 02da81c..16f132b 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticServiceImpl.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticServiceImpl.java @@ -25,15 +25,27 @@ import io.github.malonetalk.dto.semantic.BindLogicalTableRelationRequest; import io.github.malonetalk.dto.semantic.LogicalTableRelationResponse; import io.github.malonetalk.dto.semantic.RelationSemanticPageQuery; +import io.github.malonetalk.dto.semantic.RelationWorkspacePageQuery; +import io.github.malonetalk.dto.semantic.RelationWorkspaceResponse; +import io.github.malonetalk.dto.semantic.RelationWorkspaceTableResponse; +import io.github.malonetalk.dto.semantic.TableSemanticPageQuery; import io.github.malonetalk.dto.semantic.UpdateLogicalTableRelationEnabledRequest; import io.github.malonetalk.dto.semantic.UpdateLogicalTableRelationRequest; +import io.github.malonetalk.entity.ColumnInfo; import io.github.malonetalk.entity.LogicalTableRelation; +import io.github.malonetalk.entity.TableInfo; +import io.github.malonetalk.mapper.ColumnSemanticInfoMapper; import io.github.malonetalk.mapper.LogicalTableRelationMapper; +import io.github.malonetalk.mapper.TableInfoMapper; import io.github.malonetalk.service.DatasourceService; +import io.github.malonetalk.service.semantic.SemanticAvailabilityHelper; import io.github.malonetalk.utils.SemanticUtils; import java.time.LocalDateTime; +import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -43,6 +55,8 @@ public class RelationSemanticServiceImpl implements RelationSemanticService { private final DatasourceService datasourceService; + private final TableInfoMapper tableInfoMapper; + private final ColumnSemanticInfoMapper columnSemanticInfoMapper; private final LogicalTableRelationMapper logicalTableRelationMapper; private final LogicalTableRelationHelper logicalTableRelationHelper; private final SemanticConverter semanticConverter; @@ -77,6 +91,68 @@ public PageResponse getRelationPage( return PageResponse.of(items, page.getTotal(), pageNumber, pageSize); } + @Override + public RelationWorkspaceResponse getRelationWorkspace(RelationWorkspacePageQuery query) { + requireDatasource(query.datasourceId()); + int pageNumber = PageResponse.resolvePage(query.page()); + int pageSize = PageResponse.resolvePageSize(query.pageSize()); + boolean sortDescending = SemanticUtils.isDescendingSort(query.sortOrder()); + + PageHelper.startPage(pageNumber, pageSize); + Page page = + (Page) + tableInfoMapper.selectPageByDatasourceId( + new TableSemanticPageQuery( + query.datasourceId(), + pageNumber, + pageSize, + SemanticUtils.trimToNull(query.keyword()), + query.sortOrder()), + sortDescending); + if (page.isEmpty()) { + return new RelationWorkspaceResponse( + PageResponse.empty(pageNumber, pageSize), List.of()); + } + + List tableNames = page.stream().map(TableInfo::getTableName).distinct().toList(); + Set currentPageTableNames = + tableNames.stream().map(SemanticUtils::objectKey).collect(Collectors.toSet()); + Map> columnsByTableName = + columnSemanticInfoMapper + .selectByDatasourceIdAndTableNames(query.datasourceId(), tableNames) + .stream() + .collect( + Collectors.groupingBy( + column -> SemanticUtils.objectKey(column.getTableName()), + LinkedHashMap::new, + Collectors.toList())); + List nodes = + page.stream() + .map( + table -> + semanticConverter.toWorkspaceTable( + table, + columnsByTableName.getOrDefault( + SemanticUtils.objectKey( + table.getTableName()), + List.of()))) + .toList(); + List relations = + logicalTableRelationMapper + .selectByDatasourceIdAndSourceTables(query.datasourceId(), tableNames) + .stream() + .filter( + relation -> + currentPageTableNames.contains( + SemanticUtils.objectKey( + relation.getTargetTableName()))) + .map(semanticConverter::toResponse) + .toList(); + + return new RelationWorkspaceResponse( + PageResponse.of(nodes, page.getTotal(), pageNumber, pageSize), relations); + } + @Override @Transactional public LogicalTableRelationResponse createRelationSemantic( @@ -120,6 +196,9 @@ public boolean updateRelationSemanticEnabled( } LogicalTableRelation relation = requireRelation(request.datasourceId(), tableName, request.relationId()); + if (Boolean.TRUE.equals(request.enabled())) { + ensureRelationOperandsOperable(relation); + } return logicalTableRelationMapper.updateEnabled( request.relationId(), request.datasourceId(), @@ -187,6 +266,7 @@ private LogicalTableRelation buildRelation( request.targetTableName(), request.description(), request.enabled()); + ensureRelationOperandsOperable(relation); relation.setCreateTime(LocalDateTime.now()); relation.setUpdateTime(LocalDateTime.now()); return relation; @@ -204,6 +284,73 @@ private void applyRelationUpdate( request.targetTableName(), request.description(), request.enabled()); + ensureRelationOperandsOperable(relation); + } + + private void ensureRelationOperandsOperable(LogicalTableRelation relation) { + ensureTableOperable( + relation.getDatasourceId(), relation.getSourceTableName(), "sourceTable"); + ensureTableOperable( + relation.getDatasourceId(), relation.getTargetTableName(), "targetTable"); + List sourceColumns = + logicalTableRelationHelper.fromJson( + relation.getSourceColumnNamesJson(), "sourceColumnNames"); + List targetColumns = + logicalTableRelationHelper.fromJson( + relation.getTargetColumnNamesJson(), "targetColumnNames"); + ensureColumnsOperable( + relation.getDatasourceId(), + relation.getSourceTableName(), + sourceColumns, + "sourceColumnNames"); + ensureColumnsOperable( + relation.getDatasourceId(), + relation.getTargetTableName(), + targetColumns, + "targetColumnNames"); + } + + private void ensureTableOperable(Integer datasourceId, String tableName, String fieldName) { + TableInfo tableInfo = + tableInfoMapper.selectByDatasourceIdAndTableName(datasourceId, tableName); + if (tableInfo == null) { + throw new IllegalArgumentException( + fieldName + " " + tableName + " semantic metadata does not exist."); + } + if (!SemanticAvailabilityHelper.isUnavailable( + tableInfo, SemanticAvailabilityHelper.UsageLevel.USER_OPERATION)) { + return; + } + throw new IllegalArgumentException( + SemanticAvailabilityHelper.unavailableMessage( + fieldName, + tableName, + SemanticAvailabilityHelper.tableInvalidReason( + tableInfo, SemanticAvailabilityHelper.UsageLevel.USER_OPERATION))); + } + + private void ensureColumnsOperable( + Integer datasourceId, String tableName, List columnNames, String fieldName) { + for (String columnName : columnNames) { + ColumnInfo columnInfo = + columnSemanticInfoMapper.selectByDatasourceIdAndTableNameAndColumnName( + datasourceId, tableName, columnName); + if (columnInfo == null) { + throw new IllegalArgumentException( + fieldName + " " + columnName + " semantic metadata does not exist."); + } + if (!SemanticAvailabilityHelper.isUnavailable( + columnInfo, SemanticAvailabilityHelper.UsageLevel.USER_OPERATION)) { + continue; + } + throw new IllegalArgumentException( + SemanticAvailabilityHelper.unavailableMessage( + fieldName, + columnName, + SemanticAvailabilityHelper.columnInvalidReason( + columnInfo, + SemanticAvailabilityHelper.UsageLevel.USER_OPERATION))); + } } private void populateRelationFields( @@ -263,10 +410,7 @@ private LogicalTableRelation requireRelation( LogicalTableRelation relation = logicalTableRelationMapper.selectById(relationId); if (relation == null || !datasourceId.equals(relation.getDatasourceId()) - || !relation.getSourceTableName() - .equals( - SemanticUtils.trimToNotBlank(tableName, "tableName") - .toLowerCase(Locale.ROOT))) { + || !relation.getSourceTableName().equals(SemanticUtils.objectKey(tableName))) { throw new IllegalArgumentException("Logical relation does not exist."); } return relation; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncApplyService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncApplyService.java new file mode 100644 index 0000000..aa16045 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncApplyService.java @@ -0,0 +1,271 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.semantic.sync; + +import io.github.malonetalk.common.SemanticConstants; +import io.github.malonetalk.dto.semantic.SyncTableResult; +import io.github.malonetalk.entity.ColumnInfo; +import io.github.malonetalk.entity.TableInfo; +import io.github.malonetalk.mapper.ColumnSemanticInfoMapper; +import io.github.malonetalk.mapper.TableInfoMapper; +import io.github.malonetalk.utils.SemanticUtils; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import lombok.RequiredArgsConstructor; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Service; + +@Service +@RequiredArgsConstructor +public class SemanticSyncApplyService { + + private final TableInfoMapper tableInfoMapper; + private final ColumnSemanticInfoMapper columnSemanticInfoMapper; + + public TableSyncDiff ensureTablePresent( + Integer datasourceId, + String normalizedTableName, + String physicalTableDescription, + LocalDateTime now) { + TableInfo existingTable = + tableInfoMapper.selectByDatasourceIdAndTableName(datasourceId, normalizedTableName); + if (existingTable == null) { + try { + tableInfoMapper.insert( + buildNewTableInfo( + datasourceId, normalizedTableName, physicalTableDescription, now)); + return new TableSyncDiff(true, false, false); + } catch (DuplicateKeyException ignored) { + existingTable = + tableInfoMapper.selectByDatasourceIdAndTableName( + datasourceId, normalizedTableName); + } + } + if (existingTable == null) { + return new TableSyncDiff(false, false, false); + } + + boolean reactivated = Boolean.FALSE.equals(existingTable.getPhysicalStatus()); + boolean descriptionChanged = + !Objects.equals( + existingTable.getPhysicalTableDescription(), physicalTableDescription); + boolean updated = descriptionChanged; + if (reactivated || descriptionChanged) { + existingTable.setPhysicalTableDescription(physicalTableDescription); + existingTable.setPhysicalStatus(Boolean.TRUE); + existingTable.setUpdateTime(now); + tableInfoMapper.updatePhysicalCacheFields(existingTable); + } + if (reactivated) { + return new TableSyncDiff(false, true, updated); + } + if (updated) { + return new TableSyncDiff(false, false, true); + } + return new TableSyncDiff(false, false, false); + } + + public ColumnSyncDiff ensureColumnPresent( + Integer datasourceId, + String normalizedTableName, + String normalizedColumnName, + String physicalColumnDescription, + String typeName, + boolean primaryKey, + LocalDateTime now, + Map semanticColumnMap) { + ColumnInfo existingColumn = semanticColumnMap.get(normalizedColumnName); + if (existingColumn == null) { + try { + columnSemanticInfoMapper.insert( + buildNewColumnInfo( + datasourceId, + normalizedTableName, + normalizedColumnName, + physicalColumnDescription, + typeName, + primaryKey, + now)); + return new ColumnSyncDiff(true, false, false); + } catch (DuplicateKeyException ignored) { + existingColumn = + loadColumnInsertedConcurrently( + datasourceId, + normalizedTableName, + normalizedColumnName, + semanticColumnMap); + } + } + if (existingColumn == null) { + return new ColumnSyncDiff(false, false, false); + } + + boolean reactivated = Boolean.FALSE.equals(existingColumn.getPhysicalStatus()); + boolean descriptionChanged = + !Objects.equals( + existingColumn.getPhysicalColumnDescription(), physicalColumnDescription); + boolean typeChanged = !Objects.equals(existingColumn.getTypeName(), typeName); + boolean primaryKeyChanged = + !Objects.equals(existingColumn.getPrimaryKey(), Boolean.valueOf(primaryKey)); + boolean updated = descriptionChanged || typeChanged || primaryKeyChanged; + if (reactivated || updated) { + existingColumn.setPhysicalColumnDescription(physicalColumnDescription); + existingColumn.setTypeName(typeName); + existingColumn.setPrimaryKey(primaryKey); + existingColumn.setPhysicalStatus(Boolean.TRUE); + existingColumn.setUpdateTime(now); + columnSemanticInfoMapper.updatePhysicalCacheFields(existingColumn); + } + if (reactivated) { + return new ColumnSyncDiff(false, true, updated); + } + if (updated) { + return new ColumnSyncDiff(false, false, true); + } + return new ColumnSyncDiff(false, false, false); + } + + private ColumnInfo loadColumnInsertedConcurrently( + Integer datasourceId, + String normalizedTableName, + String normalizedColumnName, + Map semanticColumnMap) { + ColumnInfo existingColumn = + columnSemanticInfoMapper.selectByDatasourceIdAndTableNameAndColumnName( + datasourceId, normalizedTableName, normalizedColumnName); + if (existingColumn != null) { + semanticColumnMap.put(normalizedColumnName, existingColumn); + } + return existingColumn; + } + + public SyncTableResult markMissingTable( + Integer datasourceId, String normalizedTableName, LocalDateTime now) { + TableInfo existingTable = + tableInfoMapper.selectByDatasourceIdAndTableName(datasourceId, normalizedTableName); + if (existingTable == null) { + return new SyncTableResult( + normalizedTableName, false, false, false, false, false, 0, 0, 0, 0, "物理表不存在"); + } + + boolean tableMarkedMissing = !Boolean.FALSE.equals(existingTable.getPhysicalStatus()); + if (tableMarkedMissing) { + existingTable.setPhysicalStatus(Boolean.FALSE); + existingTable.setUpdateTime(now); + tableInfoMapper.updatePhysicalCacheFields(existingTable); + } + int missingColumnsMarked = + markAllColumnsMissing( + columnSemanticInfoMapper.selectByDatasourceIdAndTableName( + datasourceId, normalizedTableName), + now); + return new SyncTableResult( + existingTable.getTableName(), + false, + false, + false, + false, + tableMarkedMissing, + 0, + 0, + 0, + missingColumnsMarked, + "物理表不存在"); + } + + public int markMissingColumns( + List semanticColumns, Set physicalColumnNames, LocalDateTime now) { + int missingColumnsMarked = 0; + for (ColumnInfo semanticColumn : semanticColumns) { + if (physicalColumnNames.contains( + SemanticUtils.normalizeObjectName(semanticColumn.getColumnName()))) { + continue; + } + if (Boolean.FALSE.equals(semanticColumn.getPhysicalStatus())) { + continue; + } + semanticColumn.setPhysicalStatus(Boolean.FALSE); + semanticColumn.setUpdateTime(now); + columnSemanticInfoMapper.updatePhysicalCacheFields(semanticColumn); + missingColumnsMarked++; + } + return missingColumnsMarked; + } + + private int markAllColumnsMissing(List semanticColumns, LocalDateTime now) { + int missingColumnsMarked = 0; + for (ColumnInfo semanticColumn : semanticColumns) { + if (Boolean.FALSE.equals(semanticColumn.getPhysicalStatus())) { + continue; + } + semanticColumn.setPhysicalStatus(Boolean.FALSE); + semanticColumn.setUpdateTime(now); + columnSemanticInfoMapper.updatePhysicalCacheFields(semanticColumn); + missingColumnsMarked++; + } + return missingColumnsMarked; + } + + private TableInfo buildNewTableInfo( + Integer datasourceId, + String normalizedTableName, + String physicalTableDescription, + LocalDateTime now) { + TableInfo tableInfo = new TableInfo(); + tableInfo.setDatasourceId(datasourceId); + tableInfo.setTableName(normalizedTableName); + tableInfo.setPhysicalTableDescription(physicalTableDescription); + tableInfo.setTableDescription(null); + tableInfo.setDomain(SemanticConstants.DEFAULT_DOMAIN); + tableInfo.setIsVisible(Boolean.TRUE); + tableInfo.setPhysicalStatus(Boolean.TRUE); + tableInfo.setCreateTime(now); + tableInfo.setUpdateTime(now); + return tableInfo; + } + + private ColumnInfo buildNewColumnInfo( + Integer datasourceId, + String normalizedTableName, + String normalizedColumnName, + String physicalColumnDescription, + String typeName, + boolean primaryKey, + LocalDateTime now) { + ColumnInfo columnInfo = new ColumnInfo(); + columnInfo.setDatasourceId(datasourceId); + columnInfo.setTableName(normalizedTableName); + columnInfo.setColumnName(normalizedColumnName); + columnInfo.setPhysicalColumnDescription(physicalColumnDescription); + columnInfo.setTypeName(typeName); + columnInfo.setPrimaryKey(primaryKey); + columnInfo.setColumnDescription(null); + columnInfo.setIsVisible(Boolean.TRUE); + columnInfo.setPhysicalStatus(Boolean.TRUE); + columnInfo.setCreateTime(now); + columnInfo.setUpdateTime(now); + return columnInfo; + } + + public record TableSyncDiff(boolean added, boolean reactivated, boolean updated) {} + + public record ColumnSyncDiff(boolean added, boolean reactivated, boolean updated) {} +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncResultService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncResultService.java new file mode 100644 index 0000000..0de1e57 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncResultService.java @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.semantic.sync; + +import io.github.malonetalk.dto.semantic.SyncTableResult; +import io.github.malonetalk.dto.semantic.SyncTableSemanticsResponse; +import java.util.List; +import org.springframework.stereotype.Service; + +@Service +public class SemanticSyncResultService { + + public SyncTableSemanticsResponse summarize(List results) { + int addedTables = 0; + int reactivatedTables = 0; + int updatedTables = 0; + int missingTablesMarked = 0; + int addedColumns = 0; + int reactivatedColumns = 0; + int updatedColumns = 0; + int missingColumnsMarked = 0; + + for (SyncTableResult result : results) { + if (result.tableAdded()) { + addedTables++; + } + if (result.tableReactivated()) { + reactivatedTables++; + } + if (result.tableUpdated()) { + updatedTables++; + } + if (result.tableMarkedMissing()) { + missingTablesMarked++; + } + addedColumns += result.addedColumns(); + reactivatedColumns += result.reactivatedColumns(); + updatedColumns += result.updatedColumns(); + missingColumnsMarked += result.missingColumnsMarked(); + } + + return new SyncTableSemanticsResponse( + addedTables, + reactivatedTables, + updatedTables, + missingTablesMarked, + addedColumns, + reactivatedColumns, + updatedColumns, + missingColumnsMarked, + results); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncService.java new file mode 100644 index 0000000..dbd57aa --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncService.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.semantic.sync; + +import io.github.malonetalk.dto.pagination.PageResponse; +import io.github.malonetalk.dto.semantic.PhysicalTableCandidatePageQuery; +import io.github.malonetalk.dto.semantic.PhysicalTableCandidateResponse; +import io.github.malonetalk.dto.semantic.RefreshPhysicalStatusRequest; +import io.github.malonetalk.dto.semantic.SyncTableSemanticsRequest; +import io.github.malonetalk.dto.semantic.SyncTableSemanticsResponse; + +public interface SemanticSyncService { + + PageResponse getPhysicalTableCandidates( + PhysicalTableCandidatePageQuery query); + + SyncTableSemanticsResponse syncTables(SyncTableSemanticsRequest request); + + SyncTableSemanticsResponse refreshPhysicalStatus(RefreshPhysicalStatusRequest request); +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncServiceImpl.java new file mode 100644 index 0000000..a6345ee --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/sync/SemanticSyncServiceImpl.java @@ -0,0 +1,328 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.semantic.sync; + +import com.github.pagehelper.Page; +import com.github.pagehelper.PageHelper; +import io.github.malonetalk.agent.datasource.SchemaReader; +import io.github.malonetalk.dto.datasource.PhysicalColumnInfo; +import io.github.malonetalk.dto.datasource.PhysicalTableInfo; +import io.github.malonetalk.dto.pagination.PageResponse; +import io.github.malonetalk.dto.semantic.PhysicalTableCandidatePageQuery; +import io.github.malonetalk.dto.semantic.PhysicalTableCandidateResponse; +import io.github.malonetalk.dto.semantic.RefreshPhysicalStatusRequest; +import io.github.malonetalk.dto.semantic.SyncTableResult; +import io.github.malonetalk.dto.semantic.SyncTableSemanticsRequest; +import io.github.malonetalk.dto.semantic.SyncTableSemanticsResponse; +import io.github.malonetalk.entity.ColumnInfo; +import io.github.malonetalk.entity.Datasource; +import io.github.malonetalk.entity.TableInfo; +import io.github.malonetalk.mapper.ColumnSemanticInfoMapper; +import io.github.malonetalk.mapper.TableInfoMapper; +import io.github.malonetalk.service.DatasourceService; +import io.github.malonetalk.utils.SemanticUtils; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Service +@RequiredArgsConstructor +public class SemanticSyncServiceImpl implements SemanticSyncService { + + private final DatasourceService datasourceService; + private final TableInfoMapper tableInfoMapper; + private final ColumnSemanticInfoMapper columnSemanticInfoMapper; + private final SchemaReader schemaReader; + private final SemanticSyncApplyService semanticSyncApplyService; + private final SemanticSyncResultService semanticSyncResultService; + + @Override + public PageResponse getPhysicalTableCandidates( + PhysicalTableCandidatePageQuery query) { + Datasource datasource = requireDatasource(query.datasourceId()); + int pageNumber = PageResponse.resolvePage(query.page()); + int pageSize = PageResponse.resolvePageSize(query.pageSize()); + List physicalTables = schemaReader.getTables(datasource); + Map semanticTables = + tableInfoMapper.selectByDatasourceId(query.datasourceId()).stream() + .collect( + Collectors.toMap( + table -> + SemanticUtils.normalizeObjectName( + table.getTableName()), + table -> table, + (left, _right) -> left, + LinkedHashMap::new)); + String keyword = SemanticUtils.trimToNull(query.keyword()); + boolean sortDescending = SemanticUtils.isDescendingSort(query.sortOrder()); + + List filtered = + physicalTables.stream() + .filter(table -> matchesKeyword(table, keyword)) + .sorted(buildTableComparator(sortDescending)) + .map( + table -> + new PhysicalTableCandidateResponse( + table.tableName(), + SemanticUtils.trimToNull(table.remarks()), + semanticTables.containsKey( + SemanticUtils.normalizeObjectName( + table.tableName())))) + .toList(); + + int fromIndex = Math.min((pageNumber - 1) * pageSize, filtered.size()); + int toIndex = Math.min(fromIndex + pageSize, filtered.size()); + return PageResponse.of( + filtered.subList(fromIndex, toIndex), filtered.size(), pageNumber, pageSize); + } + + @Override + @Transactional + public SyncTableSemanticsResponse syncTables(SyncTableSemanticsRequest request) { + Datasource datasource = requireDatasource(request.datasourceId()); + Map physicalTables = + schemaReader.getTables(datasource).stream() + .collect( + Collectors.toMap( + table -> + SemanticUtils.normalizeObjectName( + table.tableName()), + table -> table, + (left, _right) -> left, + LinkedHashMap::new)); + Set selectedTableNames = + request.tableNames().stream() + .map(name -> SemanticUtils.trimToNotBlank(name, "tableName")) + .map(SemanticUtils::normalizeObjectName) + .collect(Collectors.toCollection(LinkedHashSet::new)); + + List results = new ArrayList<>(); + for (String normalizedTableName : selectedTableNames) { + PhysicalTableInfo physicalTable = physicalTables.get(normalizedTableName); + SyncTableResult result = + physicalTable == null + ? semanticSyncApplyService.markMissingTable( + request.datasourceId(), + normalizedTableName, + LocalDateTime.now()) + : syncSingleTable( + datasource, + request.datasourceId(), + physicalTable, + normalizedTableName); + results.add(result); + } + + return semanticSyncResultService.summarize(results); + } + + @Override + @Transactional + public SyncTableSemanticsResponse refreshPhysicalStatus(RefreshPhysicalStatusRequest request) { + Datasource datasource = requireDatasource(request.datasourceId()); + Map physicalTables = + schemaReader.getTables(datasource).stream() + .collect( + Collectors.toMap( + table -> + SemanticUtils.normalizeObjectName( + table.tableName()), + table -> table, + (left, _right) -> left, + LinkedHashMap::new)); + int pageNumber = 1; + int pageSize = PageResponse.resolvePageSize(100); + List results = new ArrayList<>(); + + while (true) { + PageHelper.startPage(pageNumber, pageSize); + Page page = + (Page) tableInfoMapper.selectByDatasourceId(request.datasourceId()); + if (page.isEmpty()) { + break; + } + for (TableInfo tableInfo : page) { + String normalizedTableName = + SemanticUtils.normalizeObjectName(tableInfo.getTableName()); + PhysicalTableInfo physicalTable = physicalTables.get(normalizedTableName); + if (physicalTable == null) { + results.add( + semanticSyncApplyService.markMissingTable( + request.datasourceId(), + normalizedTableName, + LocalDateTime.now())); + continue; + } + SyncTableResult result = + markMissingColumnsForPresentTable( + datasource, request.datasourceId(), tableInfo, physicalTable); + if (result.missingColumnsMarked() > 0) { + results.add(result); + } + } + if (page.getPageNum() >= page.getPages()) { + break; + } + pageNumber++; + } + + return semanticSyncResultService.summarize(results); + } + + private SyncTableResult markMissingColumnsForPresentTable( + Datasource datasource, + Integer datasourceId, + TableInfo tableInfo, + PhysicalTableInfo physicalTable) { + LocalDateTime now = LocalDateTime.now(); + String normalizedTableName = SemanticUtils.normalizeObjectName(tableInfo.getTableName()); + List semanticColumns = + columnSemanticInfoMapper.selectByDatasourceIdAndTableName( + datasourceId, normalizedTableName); + Set physicalColumnNames = + schemaReader.getTableSchema(datasource, physicalTable.tableName()).stream() + .map(PhysicalColumnInfo::columnName) + .map(SemanticUtils::normalizeObjectName) + .collect(Collectors.toCollection(LinkedHashSet::new)); + int missingColumnsMarked = + semanticSyncApplyService.markMissingColumns( + semanticColumns, physicalColumnNames, now); + return new SyncTableResult( + tableInfo.getTableName(), + true, + false, + false, + false, + false, + 0, + 0, + 0, + missingColumnsMarked, + "Physical column status refreshed"); + } + + private SyncTableResult syncSingleTable( + Datasource datasource, + Integer datasourceId, + PhysicalTableInfo physicalTable, + String normalizedTableName) { + LocalDateTime now = LocalDateTime.now(); + SemanticSyncApplyService.TableSyncDiff tableSyncDiff = + semanticSyncApplyService.ensureTablePresent( + datasourceId, + normalizedTableName, + SemanticUtils.trimToNull(physicalTable.remarks()), + now); + + List semanticColumns = + columnSemanticInfoMapper.selectByDatasourceIdAndTableName( + datasourceId, normalizedTableName); + Map semanticColumnMap = + semanticColumns.stream() + .collect( + Collectors.toMap( + column -> + SemanticUtils.normalizeObjectName( + column.getColumnName()), + column -> column, + (left, _right) -> left, + LinkedHashMap::new)); + List physicalColumns = + schemaReader.getTableSchema(datasource, physicalTable.tableName()); + Set physicalColumnNames = new LinkedHashSet<>(); + int addedColumns = 0; + int reactivatedColumns = 0; + int updatedColumns = 0; + + for (PhysicalColumnInfo physicalColumn : physicalColumns) { + String normalizedColumnName = + SemanticUtils.normalizeObjectName(physicalColumn.columnName()); + physicalColumnNames.add(normalizedColumnName); + SemanticSyncApplyService.ColumnSyncDiff columnSyncDiff = + semanticSyncApplyService.ensureColumnPresent( + datasourceId, + normalizedTableName, + normalizedColumnName, + SemanticUtils.trimToNull(physicalColumn.remarks()), + SemanticUtils.trimToNull(physicalColumn.typeName()), + physicalColumn.primaryKey(), + now, + semanticColumnMap); + if (columnSyncDiff.added()) { + addedColumns++; + } + if (columnSyncDiff.reactivated()) { + reactivatedColumns++; + } + if (columnSyncDiff.updated()) { + updatedColumns++; + } + } + + int missingColumnsMarked = + semanticSyncApplyService.markMissingColumns( + semanticColumns, physicalColumnNames, now); + + return new SyncTableResult( + physicalTable.tableName(), + true, + tableSyncDiff.added(), + tableSyncDiff.reactivated(), + tableSyncDiff.updated(), + false, + addedColumns, + reactivatedColumns, + updatedColumns, + missingColumnsMarked, + "同步完成"); + } + + private Datasource requireDatasource(Integer datasourceId) { + SemanticUtils.requireDatasourceId(datasourceId); + Datasource datasource = datasourceService.findById(datasourceId); + if (datasource == null) { + throw new IllegalArgumentException("Datasource does not exist: " + datasourceId); + } + return datasource; + } + + private boolean matchesKeyword(PhysicalTableInfo table, String keyword) { + if (keyword == null) { + return true; + } + return SemanticUtils.containsIgnoreCase(table.tableName(), keyword) + || SemanticUtils.containsIgnoreCase(table.remarks(), keyword); + } + + private Comparator buildTableComparator(boolean sortDescending) { + Comparator comparator = + Comparator.comparing( + table -> SemanticUtils.objectKey(table.tableName()), + Comparator.naturalOrder()); + return sortDescending ? comparator.reversed() : comparator; + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/table/TableSemanticServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/table/TableSemanticServiceImpl.java index 13c9875..a755cd6 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/table/TableSemanticServiceImpl.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/table/TableSemanticServiceImpl.java @@ -33,7 +33,6 @@ import io.github.malonetalk.utils.SemanticUtils; import java.time.LocalDateTime; import java.util.List; -import java.util.Locale; import java.util.Set; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; @@ -140,9 +139,7 @@ public List listMergedTablesByDomains( @Override public void updateTableSemantic(TableSemanticUpdateRequest request) { requireDatasource(request.datasourceId()); - String normalizedTableName = - SemanticUtils.trimToNotBlank(request.tableName(), "tableName") - .toLowerCase(Locale.ROOT); + String normalizedTableName = SemanticUtils.objectKey(request.tableName(), "tableName"); TableInfo existing = tableInfoMapper.selectByDatasourceIdAndTableName( request.datasourceId(), normalizedTableName); @@ -153,6 +150,7 @@ public void updateTableSemantic(TableSemanticUpdateRequest request) { tableInfo.setTableDescription(SemanticUtils.trimToNull(request.tableDescription())); tableInfo.setDomain(SemanticUtils.normalizeDomain(request.domain())); tableInfo.setIsVisible(request.isVisible()); + tableInfo.setPhysicalStatus(Boolean.FALSE); tableInfo.setCreateTime(LocalDateTime.now()); tableInfo.setUpdateTime(LocalDateTime.now()); tableInfoMapper.insert(tableInfo); @@ -163,7 +161,7 @@ public void updateTableSemantic(TableSemanticUpdateRequest request) { existing.setDomain(SemanticUtils.normalizeDomain(request.domain())); existing.setIsVisible(request.isVisible()); existing.setUpdateTime(LocalDateTime.now()); - tableInfoMapper.update(existing); + tableInfoMapper.updateSemanticFields(existing); } @Override @@ -186,19 +184,14 @@ public int resetTableSemantics(Integer datasourceId, List tableNames) { } Set normalizedNames = tableNames.stream() - .map( - name -> - SemanticUtils.trimToNotBlank(name, "tableName") - .toLowerCase(Locale.ROOT)) + .map(name -> SemanticUtils.objectKey(name, "tableName")) .collect(Collectors.toCollection(java.util.LinkedHashSet::new)); List matchedIds = tableInfoMapper.selectByDatasourceId(datasourceId).stream() .filter( table -> normalizedNames.contains( - SemanticUtils.trimToNotBlank( - table.getTableName(), "tableName") - .toLowerCase(Locale.ROOT))) + SemanticUtils.objectKey(table.getTableName()))) .map(TableInfo::getId) .distinct() .toList(); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/utils/SemanticUtils.java b/data-agent-backend/src/main/java/io/github/malonetalk/utils/SemanticUtils.java index 1784ad4..f8fea70 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/utils/SemanticUtils.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/utils/SemanticUtils.java @@ -66,6 +66,19 @@ public static String trimToNotBlank(String value, String label) { return value.trim(); } + public static String firstNonBlank(String... values) { + if (values == null) { + return null; + } + for (String value : values) { + String normalized = trimToNull(value); + if (normalized != null) { + return normalized; + } + } + return null; + } + public static String normalizeDomain(String domain) { String normalized = trimToNull(domain); return normalized == null @@ -73,6 +86,28 @@ public static String normalizeDomain(String domain) { : normalized.toLowerCase(Locale.ROOT); } + public static String normalizeObjectName(String value) { + return trimToNotBlank(value, "objectName").toLowerCase(Locale.ROOT); + } + + public static String objectKey(String value) { + return normalizeObjectName(value); + } + + public static String objectKey(String value, String label) { + return trimToNotBlank(value, label).toLowerCase(Locale.ROOT); + } + + public static boolean containsIgnoreCase(String value, String keyword) { + String normalizedValue = trimToNull(value); + String normalizedKeyword = trimToNull(keyword); + return normalizedValue != null + && normalizedKeyword != null + && normalizedValue + .toLowerCase(Locale.ROOT) + .contains(normalizedKeyword.toLowerCase(Locale.ROOT)); + } + public static String formatTableSchema(String tableName, List columns) { StringBuilder sb = new StringBuilder(); sb.append(String.format("Schema of table %s:%n", tableName)); diff --git a/data-agent-backend/src/main/resources/mapper/ColumnSemanticInfoMapper.xml b/data-agent-backend/src/main/resources/mapper/ColumnSemanticInfoMapper.xml index eb3cf09..ad68382 100644 --- a/data-agent-backend/src/main/resources/mapper/ColumnSemanticInfoMapper.xml +++ b/data-agent-backend/src/main/resources/mapper/ColumnSemanticInfoMapper.xml @@ -7,8 +7,12 @@ + + + + @@ -26,6 +30,16 @@ ORDER BY column_name ASC, id ASC + + + + SELECT * FROM table_info WHERE datasource_id = #{datasourceId} diff --git a/data-agent-frontend/src/api/semantic.ts b/data-agent-frontend/src/api/semantic.ts index 215c4a5..f1fd050 100644 --- a/data-agent-frontend/src/api/semantic.ts +++ b/data-agent-frontend/src/api/semantic.ts @@ -19,10 +19,10 @@ import request, { type ApiResponse } from './request'; import { getDatasourceList } from './datasource'; import type { PageResponse } from './domain'; -export interface RelationCandidateTableResponse { +export interface PhysicalTableCandidateResponse { tableName: string; - domain: string | null; - description: string | null; + physicalTableDescription: string | null; + synced: boolean; } export interface TableSemanticResponse { @@ -39,13 +39,6 @@ export interface TableSemanticResponse { export type TableSemanticInfo = TableSemanticResponse; -export interface RelationCandidateColumnResponse { - columnName: string; - description: string | null; - typeName: string | null; - primaryKey: boolean | null; -} - export interface ColumnSemanticResponse { id: number | null; columnName: string; @@ -62,6 +55,32 @@ export interface ColumnSemanticResponse { export type ColumnSemanticInfo = ColumnSemanticResponse; +export interface SyncTableResult { + tableName: string; + physicalTableFound: boolean; + tableAdded: boolean; + tableReactivated: boolean; + tableUpdated: boolean; + tableMarkedMissing: boolean; + addedColumns: number; + reactivatedColumns: number; + updatedColumns: number; + missingColumnsMarked: number; + message: string; +} + +export interface SyncTableSemanticsResponse { + addedTables: number; + reactivatedTables: number; + updatedTables: number; + missingTablesMarked: number; + addedColumns: number; + reactivatedColumns: number; + updatedColumns: number; + missingColumnsMarked: number; + results: SyncTableResult[]; +} + export interface LogicalTableRelationResponse { id: number | null; relationKey: string; @@ -79,6 +98,29 @@ export interface LogicalTableRelationResponse { updateTime: string | null; } +export interface RelationWorkspaceColumnResponse { + columnName: string; + description: string | null; + typeName: string | null; + primaryKey: boolean | null; + operable: boolean; + invalidReason: string | null; +} + +export interface RelationWorkspaceTableResponse { + tableName: string; + domain: string | null; + description: string | null; + operable: boolean; + invalidReason: string | null; + columns: RelationWorkspaceColumnResponse[]; +} + +export interface RelationWorkspaceResponse { + nodes: PageResponse; + relations: LogicalTableRelationResponse[]; +} + export interface TableSemanticPageQuery { datasourceId: number; page?: number; @@ -88,12 +130,6 @@ export interface TableSemanticPageQuery { } export type ColumnSemanticPageQuery = TableSemanticPageQuery; -export type RelationCandidateTableQuery = TableSemanticPageQuery; - -export interface RelationCandidateColumnQuery extends TableSemanticPageQuery { - tableName: string; -} - export interface LogicalRelationQuery { datasourceId: number; tableName: string; @@ -149,10 +185,13 @@ export function getTableSemanticPage(query: TableSemanticPageQuery) { }); } -export function getRelationCandidateTablePage(params: RelationCandidateTableQuery) { - return request.get>>('/semantic/tables', { - params, - }); +export function getPhysicalTableCandidatePage(query: TableSemanticPageQuery) { + return request.get>>( + '/semantic/tables/sync/candidates', + { + params: query, + }, + ); } export function getTableSemanticNames(datasourceId: number) { @@ -177,16 +216,29 @@ export function resetTableSemantic(datasourceId: number, tableName: string) { }); } -export function getColumnSemanticPage(tableName: string, query: ColumnSemanticPageQuery) { - return request.get>>( - `/semantic/tables/columns/${encodeURIComponent(tableName)}`, +export function syncTableSemantics(datasourceId: number, tableNames: string[]) { + return request.post>('/semantic/tables/sync', { + datasourceId, + tableNames, + }); +} + +export function refreshPhysicalStatus(datasourceId: number) { + return request.post>( + '/semantic/tables/sync/physical-status', + { datasourceId }, + ); +} + +export function getRelationWorkspace(query: TableSemanticPageQuery) { + return request.get>( + '/semantic/tables/relations/workspace', { params: query }, ); } -export function getRelationCandidateColumnPage(params: RelationCandidateColumnQuery) { - const { tableName, ...query } = params; - return request.get>>( +export function getColumnSemanticPage(tableName: string, query: ColumnSemanticPageQuery) { + return request.get>>( `/semantic/tables/columns/${encodeURIComponent(tableName)}`, { params: query }, ); diff --git a/data-agent-frontend/src/views/semantic/SemanticManage.vue b/data-agent-frontend/src/views/semantic/SemanticManage.vue index 20255e1..32fc79d 100644 --- a/data-agent-frontend/src/views/semantic/SemanticManage.vue +++ b/data-agent-frontend/src/views/semantic/SemanticManage.vue @@ -18,7 +18,7 @@ + + + + diff --git a/data-agent-frontend/src/views/semantic/components/TableSemanticManage.vue b/data-agent-frontend/src/views/semantic/components/TableSemanticManage.vue index ec34745..ecbf535 100644 --- a/data-agent-frontend/src/views/semantic/components/TableSemanticManage.vue +++ b/data-agent-frontend/src/views/semantic/components/TableSemanticManage.vue @@ -22,12 +22,14 @@ import { getActiveDatasourceId, getTableSemanticPage, + refreshPhysicalStatus, getTableDomains, resetTableSemantic, updateTableSemantic, type TableSemanticInfo, } from '@/api/semantic'; import ColumnSemanticManage from './ColumnSemanticManage.vue'; + import SyncPhysicalTableDialog from './SyncPhysicalTableDialog.vue'; interface TableEditForm { tableName: string; @@ -69,6 +71,9 @@ const selectedTableForColumns = ref(''); const columnManageRef = ref>(); + const syncDialogVisible = ref(false); + const refreshingPhysicalStatus = ref(false); + const ensureDatasourceId = async () => { if (datasourceId.value !== null) { return datasourceId.value; @@ -198,6 +203,54 @@ } }; + const handleOpenSync = async () => { + const activeDatasourceId = await ensureDatasourceId(); + if (activeDatasourceId === null) { + return; + } + syncDialogVisible.value = true; + }; + + const buildSyncSummary = (result: { + missingTablesMarked: number; + missingColumnsMarked: number; + }) => `标记缺失表 ${result.missingTablesMarked} 张,标记缺失列 ${result.missingColumnsMarked} 个`; + + const handleRefreshPhysicalStatus = async () => { + const activeDatasourceId = await ensureDatasourceId(); + if (activeDatasourceId === null) { + return; + } + refreshingPhysicalStatus.value = true; + try { + const response = await refreshPhysicalStatus(activeDatasourceId); + ElMessage.success(buildSyncSummary(response.data.data)); + await loadPage(); + if (columnDrawerVisible.value && selectedTableForColumns.value && columnManageRef.value) { + await columnManageRef.value.handleTableChange(selectedTableForColumns.value); + } + } catch (err) { + ElMessage.error((err as Error).message); + } finally { + refreshingPhysicalStatus.value = false; + } + }; + + const handleSyncCompleted = async (selectedTableNames: string[]) => { + await loadPage(); + const normalizedSelectedTableNameSet = new Set( + selectedTableNames.map(tableName => tableName.trim().toLowerCase()), + ); + if ( + columnDrawerVisible.value && + selectedTableForColumns.value && + normalizedSelectedTableNameSet.has(selectedTableForColumns.value.trim().toLowerCase()) && + columnManageRef.value + ) { + await columnManageRef.value.handleTableChange(selectedTableForColumns.value); + } + }; + defineExpose({ loadPage, }); @@ -214,7 +267,13 @@

表语义列表

管理和维护物理表的业务语义信息。

- 共 {{ page.total }} 张表 +
+ 共 {{ page.total }} 张表 + + 刷新物理状态 + + 同步表 +
@@ -241,6 +300,13 @@ + + + @@ -333,6 +399,12 @@ :table-name="selectedTableForColumns" /> + + @@ -345,6 +417,12 @@ margin-bottom: 20px; } + .header-actions { + display: flex; + align-items: center; + gap: 12px; + } + .section-header h3 { font-size: 16px; color: var(--app-text-primary); diff --git a/data-agent-frontend/src/views/semantic/relation/types.ts b/data-agent-frontend/src/views/semantic/types.ts similarity index 77% rename from data-agent-frontend/src/views/semantic/relation/types.ts rename to data-agent-frontend/src/views/semantic/types.ts index 5e3516a..fdad6ff 100644 --- a/data-agent-frontend/src/views/semantic/relation/types.ts +++ b/data-agent-frontend/src/views/semantic/types.ts @@ -15,11 +15,6 @@ * along with this program. If not, see . */ -import type { - RelationCandidateColumnResponse, - RelationCandidateTableResponse, -} from '@/api/semantic'; - export interface RelationForm { sourceTableName: string; sourceColumnNames: string[]; @@ -37,12 +32,29 @@ export interface RelationDraftPreview { enabled: boolean; } -export interface TableNodeLayout extends RelationCandidateTableResponse { +export interface RelationTableNode { + tableName: string; + domain: string | null; + description: string | null; + operable: boolean; + invalidReason: string | null; +} + +export interface RelationColumnNode { + columnName: string; + description: string | null; + typeName: string | null; + primaryKey: boolean | null; + operable: boolean; + invalidReason: string | null; +} + +export interface TableNodeLayout extends RelationTableNode { x: number; y: number; width: number; height: number; - columns: RelationCandidateColumnResponse[]; + columns: RelationColumnNode[]; } export interface RelationViewportState { diff --git a/sql/data_source.sql b/sql/data_source.sql index d66aae3..ec21a3a 100644 --- a/sql/data_source.sql +++ b/sql/data_source.sql @@ -20,10 +20,12 @@ CREATE TABLE IF NOT EXISTS `datasource` ( CREATE TABLE IF NOT EXISTS `table_info` ( `id` INT NOT NULL AUTO_INCREMENT COMMENT '主键ID', `table_name` VARCHAR(255) NOT NULL COMMENT '表名', + `physical_table_description` VARCHAR(500) DEFAULT NULL COMMENT '物理表原始描述', `table_description` VARCHAR(500) DEFAULT NULL COMMENT '表描述', `domain` VARCHAR(255) DEFAULT NULL COMMENT '领域', `datasource_id` INT NOT NULL COMMENT '关联数据源ID', `is_visible` TINYINT(1) DEFAULT 1 COMMENT '是否可见', + `physical_status` TINYINT(1) DEFAULT 1 COMMENT '物理表是否存在', `create_time` DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', `update_time` DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', PRIMARY KEY (`id`), @@ -39,8 +41,12 @@ CREATE TABLE IF NOT EXISTS `column_info` ( `datasource_id` INT NOT NULL COMMENT '关联数据源ID', `table_name` VARCHAR(255) NOT NULL COMMENT '表名', `column_name` VARCHAR(255) NOT NULL COMMENT '列名', + `physical_column_description` VARCHAR(500) DEFAULT NULL COMMENT '物理列原始描述', + `type_name` VARCHAR(255) DEFAULT NULL COMMENT '物理列类型', + `primary_key` TINYINT(1) DEFAULT NULL COMMENT '是否物理主键', `column_description` VARCHAR(500) DEFAULT NULL COMMENT '列描述', `is_visible` TINYINT(1) DEFAULT 1 COMMENT '是否可见', + `physical_status` TINYINT(1) DEFAULT 1 COMMENT '物理列是否存在', `create_time` DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', `update_time` DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', PRIMARY KEY (`id`),