From 1b9e28be5a79065de93c71ddc74a85ada356c2cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=BD=98=E8=8D=A3=E6=99=9F?= <9691125+pan-rongsheng@user.noreply.gitee.com> Date: Tue, 30 Sep 2025 17:35:39 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=B8=9A=E5=8A=A1=E9=80=BB?= =?UTF-8?q?=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../admin/base/MasterDataSyncController.java | 51 ++ .../admin/base/vo/MasterDataSyncReqVO.java | 31 ++ .../masterdata/MdmMaterialCategoryDO.java | 35 ++ .../masterdata/MdmMaterialViewDO.java | 35 ++ .../masterdata/MdmMaterialCategoryMapper.java | 21 + .../masterdata/MdmMaterialViewMapper.java | 27 + .../config/MasterDataSyncConfiguration.java | 24 + .../sync/config/MasterDataSyncProperties.java | 68 +++ .../constant/MasterDataSyncConstants.java | 15 + .../MasterDataCategorySyncService.java | 11 + .../MasterDataCategorySyncServiceImpl.java | 214 ++++++++ .../masterdatasync/MasterDataSyncService.java | 12 + .../MasterDataSyncServiceImpl.java | 488 ++++++++++++++++++ .../dto/MasterDataCategorySyncReport.java | 59 +++ .../dto/MasterDataSyncCommand.java | 33 ++ .../dto/MasterDataSyncReport.java | 104 ++++ .../support/MasterDataPropertyDefinition.java | 52 ++ .../MasterDataSyncNotificationPayload.java | 16 + .../support/MasterDataSyncNotifier.java | 61 +++ .../masterdata/MdmMaterialCategoryMapper.xml | 17 + .../masterdata/MdmMaterialViewMapper.xml | 71 +++ zt-module-base/主数据同步说明.md | 140 +++++ 22 files changed, 1585 insertions(+) create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/controller/admin/base/MasterDataSyncController.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/controller/admin/base/vo/MasterDataSyncReqVO.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/dal/dataobject/masterdata/MdmMaterialCategoryDO.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/dal/dataobject/masterdata/MdmMaterialViewDO.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/dal/mysql/masterdata/MdmMaterialCategoryMapper.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/dal/mysql/masterdata/MdmMaterialViewMapper.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/framework/sync/config/MasterDataSyncConfiguration.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/framework/sync/config/MasterDataSyncProperties.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/framework/sync/constant/MasterDataSyncConstants.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/MasterDataCategorySyncService.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/MasterDataCategorySyncServiceImpl.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/MasterDataSyncService.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/MasterDataSyncServiceImpl.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/dto/MasterDataCategorySyncReport.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/dto/MasterDataSyncCommand.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/dto/MasterDataSyncReport.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/support/MasterDataPropertyDefinition.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/support/MasterDataSyncNotificationPayload.java create mode 100644 zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/support/MasterDataSyncNotifier.java create mode 100644 zt-module-base/zt-module-base-server/src/main/resources/mapper/masterdata/MdmMaterialCategoryMapper.xml create mode 100644 zt-module-base/zt-module-base-server/src/main/resources/mapper/masterdata/MdmMaterialViewMapper.xml create mode 100644 zt-module-base/主数据同步说明.md diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/controller/admin/base/MasterDataSyncController.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/controller/admin/base/MasterDataSyncController.java new file mode 100644 index 00000000..be85284f --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/controller/admin/base/MasterDataSyncController.java @@ -0,0 +1,51 @@ +package com.zt.plat.module.base.controller.admin.base; + +import com.zt.plat.framework.common.pojo.CommonResult; +import com.zt.plat.framework.common.util.object.BeanUtils; +import com.zt.plat.module.base.controller.admin.base.vo.MasterDataSyncReqVO; +import com.zt.plat.module.base.service.masterdatasync.MasterDataCategorySyncService; +import com.zt.plat.module.base.service.masterdatasync.MasterDataSyncService; +import com.zt.plat.module.base.service.masterdatasync.dto.MasterDataSyncCommand; +import com.zt.plat.module.base.service.masterdatasync.dto.MasterDataSyncReport; +import com.zt.plat.module.base.service.masterdatasync.dto.MasterDataCategorySyncReport; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.annotation.Resource; +import jakarta.validation.Valid; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import static com.zt.plat.framework.common.pojo.CommonResult.success; + +@Tag(name = "管理后台 - 主数据同步") +@RestController +@RequestMapping("/base/master-data-sync") +@Validated +public class MasterDataSyncController { + + @Resource + private MasterDataSyncService masterDataSyncService; + @Resource + private MasterDataCategorySyncService masterDataCategorySyncService; + + @PostMapping("/execute") + @Operation(summary = "执行主数据同步") + @PreAuthorize("@ss.hasPermission('base:master-data-sync:execute')") + public CommonResult execute(@Valid @RequestBody MasterDataSyncReqVO reqVO) { + MasterDataSyncCommand command = BeanUtils.toBean(reqVO, MasterDataSyncCommand.class); + MasterDataSyncReport report = masterDataSyncService.sync(command); + return success(report); + } + + @PostMapping("/categories") + @Operation(summary = "同步物料分类") + @PreAuthorize("@ss.hasPermission('base:master-data-sync:categories')") + public CommonResult syncCategories() { + MasterDataCategorySyncReport report = masterDataCategorySyncService.syncAll(); + return success(report); + } +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/controller/admin/base/vo/MasterDataSyncReqVO.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/controller/admin/base/vo/MasterDataSyncReqVO.java new file mode 100644 index 00000000..a1559f9a --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/controller/admin/base/vo/MasterDataSyncReqVO.java @@ -0,0 +1,31 @@ +package com.zt.plat.module.base.controller.admin.base.vo; + +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.Positive; +import lombok.Data; +import org.springframework.format.annotation.DateTimeFormat; + +import java.time.LocalDateTime; +import java.util.List; + +@Data +@Schema(description = "主数据同步请求参数") +public class MasterDataSyncReqVO { + + @Schema(description = "增量同步的起始记录时间，留空执行全量") + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private LocalDateTime since; + + @Schema(description = "拉取批大小，不填写则使用配置默认值") + @Positive(message = "batchSize 必须为正数") + private Integer batchSize; + + @Schema(description = "指定要刷新同步的物料编码列表") + private List materialCodes; + + @Schema(description = "可选的本次同步记录数上限，未填写表示不限制") + @Positive(message = "recordLimit 必须为正数") + private Long recordLimit; +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/dal/dataobject/masterdata/MdmMaterialCategoryDO.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/dal/dataobject/masterdata/MdmMaterialCategoryDO.java new file mode 100644 index 00000000..78d8954f --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/dal/dataobject/masterdata/MdmMaterialCategoryDO.java @@ -0,0 +1,35 @@ +package com.zt.plat.module.base.dal.dataobject.masterdata; + +import lombok.Data; + +/** + * 外部 MDM 分类视图的数据对象。 + */ +@Data +public class MdmMaterialCategoryDO { + + /** + * 分类主键 ID（CODEID）。 + */ + private Long codeId; + + /** + * 分类编码，例如 01 / 0101 / 010101。 + */ + private String code; + + /** + * 分类名称（DESC1）。 + */ + private String name; + + /** + * 备注信息（DESC2）。 + */ + private String remark; + + /** + * 父级分类主键 ID（PARENTID）。 + */ + private Long parentCodeId; +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/dal/dataobject/masterdata/MdmMaterialViewDO.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/dal/dataobject/masterdata/MdmMaterialViewDO.java new file mode 100644 index 00000000..ce5b59ac --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/dal/dataobject/masterdata/MdmMaterialViewDO.java @@ -0,0 +1,35 @@ +package com.zt.plat.module.base.dal.dataobject.masterdata; + +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * 外部 MDM MySQL 物料视图的投影，用于承接同步字段。 + */ +@Data +public class MdmMaterialViewDO { + + private Long codeId; + private String materialCode; + private String categoryCode; + private String categoryName; + private String materialName; + private String baseUnitCode; + private String baseUnitName; + private String shortDescription; + private String longDescription; + private String chalcoCode; + private String majorClassCode; + private String majorClassName; + private String specification; + private String model; + private String texture; + private String drawingNumber; + private String orderNumber; + private String otherParameters; + private String equipmentCategory; + private String manufacturer; + private String source; + private LocalDateTime recordTime; +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/dal/mysql/masterdata/MdmMaterialCategoryMapper.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/dal/mysql/masterdata/MdmMaterialCategoryMapper.java new file mode 100644 index 00000000..ec8770ae --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/dal/mysql/masterdata/MdmMaterialCategoryMapper.java @@ -0,0 +1,21 @@ +package com.zt.plat.module.base.dal.mysql.masterdata; + +import com.baomidou.dynamic.datasource.annotation.DS; +import com.zt.plat.module.base.dal.dataobject.masterdata.MdmMaterialCategoryDO; +import com.zt.plat.module.base.framework.sync.constant.MasterDataSyncConstants; +import org.apache.ibatis.annotations.Mapper; + +import java.util.List; + +/** + * 读取外部 MDM 分类视图的 Mapper。 + */ +@Mapper +@DS(MasterDataSyncConstants.DEFAULT_SOURCE_DATASOURCE) +public interface MdmMaterialCategoryMapper { + + /** + * 拉取全部分类档案。 + */ + List selectAll(); +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/dal/mysql/masterdata/MdmMaterialViewMapper.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/dal/mysql/masterdata/MdmMaterialViewMapper.java new file mode 100644 index 00000000..42d88f1c --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/dal/mysql/masterdata/MdmMaterialViewMapper.java @@ -0,0 +1,27 @@ +package com.zt.plat.module.base.dal.mysql.masterdata; + +import com.baomidou.dynamic.datasource.annotation.DS; +import com.zt.plat.module.base.dal.dataobject.masterdata.MdmMaterialViewDO; +import com.zt.plat.module.base.framework.sync.constant.MasterDataSyncConstants; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.time.LocalDateTime; +import java.util.Collection; +import java.util.List; + +/** + * Mapper that exposes read-only access to the external MDM material view. + */ +@Mapper +@DS(MasterDataSyncConstants.DEFAULT_SOURCE_DATASOURCE) +public interface MdmMaterialViewMapper { + + List selectSlice(@Param("offset") long offset, + @Param("limit") int limit, + @Param("since") LocalDateTime since, + @Param("codes") Collection codes); + + Long countEligible(@Param("since") LocalDateTime since, + @Param("codes") Collection codes); +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/framework/sync/config/MasterDataSyncConfiguration.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/framework/sync/config/MasterDataSyncConfiguration.java new file mode 100644 index 00000000..41fa787a --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/framework/sync/config/MasterDataSyncConfiguration.java @@ -0,0 +1,24 @@ +package com.zt.plat.module.base.framework.sync.config; + +import okhttp3.OkHttpClient; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * 主数据同步用到的基础配置。 + */ +@Configuration +@EnableConfigurationProperties(MasterDataSyncProperties.class) +public class MasterDataSyncConfiguration { + + @Bean + public OkHttpClient masterDataSyncOkHttpClient(MasterDataSyncProperties properties) { + MasterDataSyncProperties.HttpProperties http = properties.getHttp(); + return new OkHttpClient.Builder() + .connectTimeout(http.getConnectTimeout()) + .readTimeout(http.getReadTimeout()) + .writeTimeout(http.getWriteTimeout()) + .build(); + } +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/framework/sync/config/MasterDataSyncProperties.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/framework/sync/config/MasterDataSyncProperties.java new file mode 100644 index 00000000..2ef02810 --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/framework/sync/config/MasterDataSyncProperties.java @@ -0,0 +1,68 @@ +package com.zt.plat.module.base.framework.sync.config; + +import com.zt.plat.module.base.framework.sync.constant.MasterDataSyncConstants; +import jakarta.validation.constraints.Min; +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; + +/** + * 主数据同步管道的配置项。 + */ +@Data +@Validated +@ConfigurationProperties(prefix = "base.master-data-sync") +public class MasterDataSyncProperties { + + /** + * 是否允许运行主数据同步。 + */ + private boolean enabled = true; + + /** + * 从外部数据源读取时的默认批量大小。 + */ + @Min(1) + private int batchSize = 500; + + /** + * 指向外部 MDM MySQL 的动态数据源名称。 + */ + private String sourceDatasourceName = MasterDataSyncConstants.DEFAULT_SOURCE_DATASOURCE; + + /** + * 成功是否进行回调通知。 + */ + private boolean notifyOnSuccess = true; + + /** + * 失败是否进行回调通知。 + */ + private boolean notifyOnFailure = true; + + /** + * 可选的回调地址，用于推送同步结果。 + */ + private String callbackUrl; + + /** + * 回调请求需要附加的静态请求头。 + */ + private Map callbackHeaders = new HashMap<>(); + + /** + * HTTP 客户端的超时时间等参数。 + */ + private final HttpProperties http = new HttpProperties(); + + @Data + public static class HttpProperties { + private Duration connectTimeout = Duration.ofSeconds(5); + private Duration readTimeout = Duration.ofSeconds(30); + private Duration writeTimeout = Duration.ofSeconds(30); + } +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/framework/sync/constant/MasterDataSyncConstants.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/framework/sync/constant/MasterDataSyncConstants.java new file mode 100644 index 00000000..4032604d --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/framework/sync/constant/MasterDataSyncConstants.java @@ -0,0 +1,15 @@ +package com.zt.plat.module.base.framework.sync.constant; + +/** + * 主数据同步相关的通用常量。 + */ +public final class MasterDataSyncConstants { + + private MasterDataSyncConstants() { + } + + /** + * 指向外部 MDM MySQL 的默认数据源名称。 + */ + public static final String DEFAULT_SOURCE_DATASOURCE = "mdm"; +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/MasterDataCategorySyncService.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/MasterDataCategorySyncService.java new file mode 100644 index 00000000..3e33148d --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/MasterDataCategorySyncService.java @@ -0,0 +1,11 @@ +package com.zt.plat.module.base.service.masterdatasync; + +import com.zt.plat.module.base.service.masterdatasync.dto.MasterDataCategorySyncReport; + +/** + * 物料分类同步服务接口。 + */ +public interface MasterDataCategorySyncService { + + MasterDataCategorySyncReport syncAll(); +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/MasterDataCategorySyncServiceImpl.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/MasterDataCategorySyncServiceImpl.java new file mode 100644 index 00000000..0c0a7e07 --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/MasterDataCategorySyncServiceImpl.java @@ -0,0 +1,214 @@ +package com.zt.plat.module.base.service.masterdatasync; + +import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.util.StrUtil; +import com.zt.plat.framework.common.exception.util.ServiceExceptionUtil; +import com.zt.plat.module.base.dal.dao.materialclasses.MaterialClassesMapper; +import com.zt.plat.module.base.dal.dataobject.materialclasses.MaterialClassesDO; +import com.zt.plat.module.base.dal.dataobject.masterdata.MdmMaterialCategoryDO; +import com.zt.plat.module.base.dal.mysql.masterdata.MdmMaterialCategoryMapper; +import com.zt.plat.module.base.enums.ErrorCodeConstants; +import com.zt.plat.module.base.framework.sync.config.MasterDataSyncProperties; +import com.zt.plat.module.base.service.masterdatasync.dto.MasterDataCategorySyncReport; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * 物料分类同步实现。 + */ +@Slf4j +@Service +public class MasterDataCategorySyncServiceImpl implements MasterDataCategorySyncService { + + private static final long ROOT_CLASS_PARENT_ID = 0L; + + @Resource + private MdmMaterialCategoryMapper mdmMaterialCategoryMapper; + @Resource + private MaterialClassesMapper materialClassesMapper; + @Resource + private MasterDataSyncProperties properties; + + @Override + public MasterDataCategorySyncReport syncAll() { + if (!properties.isEnabled()) { + throw ServiceExceptionUtil.exception(ErrorCodeConstants.MASTER_DATA_SYNC_DISABLED); + } + MasterDataCategorySyncReport report = MasterDataCategorySyncReport.start(); + try { + List categories = mdmMaterialCategoryMapper.selectAll(); + if (CollUtil.isEmpty(categories)) { + report.markSuccess(); + return report; + } + categories.sort(Comparator.comparing(MdmMaterialCategoryDO::getCode)); + Map classesByCode = loadExistingClasses(categories); + Map classesById = classesByCode.values().stream() + .filter(item -> item.getId() != null) + .collect(Collectors.toMap(MaterialClassesDO::getId, item -> item, (a, b) -> a, LinkedHashMap::new)); + // 统一收集新增分类，批量写入提升性能 + List pendingClassInserts = new ArrayList<>(); + for (MdmMaterialCategoryDO category : categories) { + Long sourceId = category.getCodeId(); + if (sourceId == null) { + log.warn("分类 {} 缺少主键 CODEID，已跳过", category.getCode()); + continue; + } + String code = normalizeCode(category.getCode()); + if (code == null) { + continue; + } + report.incrementProcessed(1); + long level = determineLevel(code); + Long parentId = resolveParentId(category, classesByCode, classesById, report); + if (parentId == null) { + parentId = ROOT_CLASS_PARENT_ID; + } + String name = StrUtil.emptyIfNull(StrUtil.trim(category.getName())); + String remark = StrUtil.emptyIfNull(StrUtil.trim(category.getRemark())); + MaterialClassesDO current = classesByCode.get(code); + if (current == null) { + MaterialClassesDO created = MaterialClassesDO.builder() + .id(sourceId) + .code(code) + .name(name) + .level(level) + .parentId(parentId) + .remark(remark) + .build(); + pendingClassInserts.add(created); + classesByCode.put(code, created); + if (created.getId() != null) { + classesById.put(created.getId(), created); + } + report.incrementInserted(); + } else { + if (current.getId() == null && sourceId != null) { + current.setId(sourceId); + classesById.put(sourceId, current); + } + boolean needUpdate = false; + MaterialClassesDO update = new MaterialClassesDO(); + update.setId(current.getId()); + if (!Objects.equals(current.getParentId(), parentId)) { + update.setParentId(parentId); + current.setParentId(parentId); + needUpdate = true; + } + if (StrUtil.isNotBlank(name) && !StrUtil.equals(current.getName(), name)) { + update.setName(name); + current.setName(name); + needUpdate = true; + } + if (!Objects.equals(current.getLevel(), level)) { + update.setLevel(level); + current.setLevel(level); + needUpdate = true; + } + if (!Objects.equals(StrUtil.emptyIfNull(current.getRemark()), remark)) { + update.setRemark(remark); + current.setRemark(remark); + needUpdate = true; + } + if (needUpdate) { + materialClassesMapper.updateById(update); + report.incrementUpdated(); + } + } + } + if (CollUtil.isNotEmpty(pendingClassInserts)) { + materialClassesMapper.insertBatch(pendingClassInserts); + } + report.markSuccess(); + return report; + } catch (Throwable ex) { + report.markFailure(ex.getMessage()); + throw ex; + } finally { + report.finish(); + } + } + + private Map loadExistingClasses(List categories) { + Set codes = categories.stream() + .map(MdmMaterialCategoryDO::getCode) + .map(this::normalizeCode) + .filter(Objects::nonNull) + .collect(Collectors.toCollection(LinkedHashSet::new)); + List existing = materialClassesMapper.selectByCodes(codes); + Map result = new LinkedHashMap<>(); + if (existing != null) { + existing.forEach(item -> { + String key = normalizeCode(item.getCode()); + if (key != null) { + result.put(key, item); + } + }); + } + return result; + } + + private Long resolveParentId(MdmMaterialCategoryDO category, + Map classesByCode, + Map classesById, + MasterDataCategorySyncReport report) { + if (category.getParentCodeId() != null && category.getParentCodeId() > 0) { + MaterialClassesDO parent = classesById.get(category.getParentCodeId()); + if (parent != null) { + return parent.getId(); + } + log.warn("分类 {} 指定父级 ID {} 不存在，将尝试按编码推断", category.getCode(), category.getParentCodeId()); + } + String parentCode = resolveParentCode(normalizeCode(category.getCode())); + if (parentCode == null) { + return ROOT_CLASS_PARENT_ID; + } + MaterialClassesDO parent = classesByCode.get(parentCode); + if (parent == null) { + log.warn("分类 {} 缺失父级 {}，将挂载到根节点", category.getCode(), parentCode); + if (report != null) { + report.incrementMissingParent(); + } + return null; + } + return parent.getId(); + } + + private String resolveParentCode(String code) { + if (code == null) { + return null; + } + if (code.length() <= 2) { + return null; + } + if (code.length() <= 4) { + return code.substring(0, 2); + } + return code.substring(0, 4); + } + + private long determineLevel(String code) { + if (code.length() <= 2) { + return 1L; + } + if (code.length() <= 4) { + return 2L; + } + return 3L; + } + + private String normalizeCode(String code) { + return StrUtil.isBlank(code) ? null : StrUtil.trim(code); + } +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/MasterDataSyncService.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/MasterDataSyncService.java new file mode 100644 index 00000000..3ffadd3f --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/MasterDataSyncService.java @@ -0,0 +1,12 @@ +package com.zt.plat.module.base.service.masterdatasync; + +import com.zt.plat.module.base.service.masterdatasync.dto.MasterDataSyncCommand; +import com.zt.plat.module.base.service.masterdatasync.dto.MasterDataSyncReport; + +/** + * 物料主数据同步的服务入口。 + */ +public interface MasterDataSyncService { + + MasterDataSyncReport sync(MasterDataSyncCommand command); +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/MasterDataSyncServiceImpl.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/MasterDataSyncServiceImpl.java new file mode 100644 index 00000000..3fd3f984 --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/MasterDataSyncServiceImpl.java @@ -0,0 +1,488 @@ +package com.zt.plat.module.base.service.masterdatasync; + +import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.util.StrUtil; +import com.baomidou.mybatisplus.core.toolkit.IdWorker; +import com.zt.plat.framework.common.exception.util.ServiceExceptionUtil; +import com.zt.plat.module.base.dal.dao.materialclasses.MaterialClassesMapper; +import com.zt.plat.module.base.dal.dao.materialhasclasses.MaterialHasClassesMapper; +import com.zt.plat.module.base.dal.dao.materialhasproperties.MaterialHasPropertiesMapper; +import com.zt.plat.module.base.dal.dao.materialproperties.MaterialPropertiesMapper; +import com.zt.plat.module.base.dal.dataobject.base.MaterialInfomationDO; +import com.zt.plat.module.base.dal.dataobject.materialclasses.MaterialClassesDO; +import com.zt.plat.module.base.dal.dataobject.materialhasclasses.MaterialHasClassesDO; +import com.zt.plat.module.base.dal.dataobject.materialhasproperties.MaterialHasPropertiesDO; +import com.zt.plat.module.base.dal.dataobject.materialproperties.MaterialPropertiesDO; +import com.zt.plat.module.base.dal.dataobject.masterdata.MdmMaterialViewDO; +import com.zt.plat.module.base.dal.mysql.base.MaterialInfomationMapper; +import com.zt.plat.module.base.dal.mysql.masterdata.MdmMaterialViewMapper; +import com.zt.plat.module.base.enums.ErrorCodeConstants; +import com.zt.plat.module.base.framework.sync.config.MasterDataSyncProperties; +import com.zt.plat.module.base.service.masterdatasync.dto.MasterDataSyncCommand; +import com.zt.plat.module.base.service.masterdatasync.dto.MasterDataSyncReport; +import com.zt.plat.module.base.service.masterdatasync.support.MasterDataPropertyDefinition; +import com.zt.plat.module.base.service.masterdatasync.support.MasterDataSyncNotifier; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionTemplate; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * 主数据同步核心实现，负责从外部 MDM 视图读取数据并落库到本地 BSE 表。 + */ +@Slf4j +@Service +public class MasterDataSyncServiceImpl implements MasterDataSyncService { + + private static final long ROOT_CLASS_PARENT_ID = 0L; + + @Resource + private MdmMaterialViewMapper mdmMaterialViewMapper; + @Resource + private MaterialInfomationMapper materialInfomationMapper; + @Resource + private MaterialClassesMapper materialClassesMapper; + @Resource + private MaterialHasClassesMapper materialHasClassesMapper; + @Resource + private MaterialPropertiesMapper materialPropertiesMapper; + @Resource + private MaterialHasPropertiesMapper materialHasPropertiesMapper; + @Resource + private TransactionTemplate transactionTemplate; + @Resource + private MasterDataSyncProperties properties; + @Resource + private MasterDataSyncNotifier notifier; + + @Override + public MasterDataSyncReport sync(MasterDataSyncCommand command) { + if (!properties.isEnabled()) { + throw ServiceExceptionUtil.exception(ErrorCodeConstants.MASTER_DATA_SYNC_DISABLED); + } + MasterDataSyncReport report = MasterDataSyncReport.start(command); + List materialCodes = sanitizeCodes(command.getMaterialCodes()); + report.setMaterialCodes(materialCodes); + int batchSize = resolveBatchSize(command.getBatchSize()); + report.setBatchSize(batchSize); + Long recordLimit = resolveRecordLimit(command.getRecordLimit()); + report.setRecordLimit(recordLimit); + LocalDateTime since = command.getSince(); + Map propertyDefinitions = ensurePropertyDefinitions(report); + // 缓存已读取的物料与分类，避免批次间重复查询数据库 + Map materialCacheById = new LinkedHashMap<>(); + Map materialCacheByCode = new LinkedHashMap<>(); + Map categoryCacheByCode = new LinkedHashMap<>(); + + long offset = 0L; + Throwable failure = null; + try { + while (true) { + int fetchSize = batchSize; + if (recordLimit != null) { + long remaining = recordLimit - report.getProcessedRecords(); + if (remaining <= 0) { + break; + } + if (remaining < fetchSize) { + fetchSize = (int) remaining; + } + } + List slice = mdmMaterialViewMapper.selectSlice(offset, fetchSize, since, materialCodes); + if (CollUtil.isEmpty(slice)) { + break; + } + offset += slice.size(); + report.incrementProcessedRecords(slice.size()); + final List batch = slice; + transactionTemplate.executeWithoutResult(status -> processBatch(batch, propertyDefinitions, report, + materialCacheById, materialCacheByCode, categoryCacheByCode)); + } + report.markSuccess(); + return report; + } catch (Throwable ex) { + failure = ex; + report.markFailure(ex.getMessage()); + throw ex; + } finally { + report.finish(); + notifier.dispatch(report, failure); + } + } + + private List sanitizeCodes(List codes) { + if (CollUtil.isEmpty(codes)) { + return Collections.emptyList(); + } + return codes.stream() + .filter(StrUtil::isNotBlank) + .map(code -> StrUtil.trim(code).toUpperCase()) + .distinct() + .collect(Collectors.toList()); + } + + private int resolveBatchSize(Integer requested) { + if (requested != null && requested > 0) { + return requested; + } + return Math.max(1, properties.getBatchSize()); + } + + private Long resolveRecordLimit(Long requested) { + if (requested == null || requested <= 0) { + return null; + } + return requested; + } + + /** + * 确保所有需要的属性定义已经存在，若缺失则自动创建。 + */ + private Map ensurePropertyDefinitions(MasterDataSyncReport report) { + List codes = Arrays.stream(MasterDataPropertyDefinition.values()) + .map(MasterDataPropertyDefinition::getCode) + .toList(); + List existing = materialPropertiesMapper.selectByCodes(codes); + Map existingMap = Optional.ofNullable(existing) + .orElse(Collections.emptyList()) + .stream() + .collect(Collectors.toMap(MaterialPropertiesDO::getCode, Function.identity())); + + Map mapping = new EnumMap<>(MasterDataPropertyDefinition.class); + List pendingInsertDefinitions = new ArrayList<>(); + for (MasterDataPropertyDefinition definition : MasterDataPropertyDefinition.values()) { + MaterialPropertiesDO current = existingMap.get(definition.getCode()); + if (current == null) { + current = MaterialPropertiesDO.builder() + .id(IdWorker.getId()) + .code(definition.getCode()) + .name(definition.getDisplayName()) + .dataType(definition.getDataType()) + .remark("MDM同步自动创建") + .build(); + pendingInsertDefinitions.add(current); + report.incrementInsertedPropertyDefinitions(); + } else { + boolean needUpdate = false; + if (!StrUtil.equals(current.getName(), definition.getDisplayName())) { + current.setName(definition.getDisplayName()); + needUpdate = true; + } + if (!StrUtil.equals(current.getDataType(), definition.getDataType())) { + current.setDataType(definition.getDataType()); + needUpdate = true; + } + if (needUpdate) { + MaterialPropertiesDO update = new MaterialPropertiesDO(); + update.setId(current.getId()); + update.setName(current.getName()); + update.setDataType(current.getDataType()); + materialPropertiesMapper.updateById(update); + report.incrementUpdatedPropertyDefinitions(); + } + } + mapping.put(definition, current); + } + if (CollUtil.isNotEmpty(pendingInsertDefinitions)) { + materialPropertiesMapper.insertBatch(pendingInsertDefinitions); + } + return mapping; + } + + private void processBatch(List batch, + Map propertyDefinitions, + MasterDataSyncReport report, + Map materialCacheById, + Map materialCacheByCode, + Map categoryCacheByCode) { + if (CollUtil.isEmpty(batch)) { + return; + } + // 1. 落库物料主数据 + Map materials = upsertMaterials(batch, report, materialCacheById, materialCacheByCode); + // 2. 读取已存在的分类信息 + Map categoryClasses = loadCategoryClasses(batch, categoryCacheByCode); + // 3. 建立物料与分类的关系 + bindMaterialClasses(batch, materials, categoryClasses, report); + // 4. 写入物料属性及属性值 + upsertMaterialProperties(batch, materials, propertyDefinitions, report); + } + + private Map loadCategoryClasses(List batch, + Map categoryCacheByCode) { + Set requestedCodes = batch.stream() + .map(MdmMaterialViewDO::getCategoryCode) + .map(this::normalizeValue) + .filter(Objects::nonNull) + .collect(Collectors.toCollection(LinkedHashSet::new)); + if (requestedCodes.isEmpty()) { + return Collections.emptyMap(); + } + Map mapping = requestedCodes.stream() + .map(code -> new Object[]{code, categoryCacheByCode.get(code)}) + .filter(pair -> pair[1] != null) + .collect(Collectors.toMap(pair -> (String) pair[0], pair -> (MaterialClassesDO) pair[1], (a, b) -> a, LinkedHashMap::new)); + Set missing = new LinkedHashSet<>(requestedCodes); + missing.removeIf(mapping::containsKey); + if (!missing.isEmpty()) { + List newlyLoaded = materialClassesMapper.selectByCodes(missing); + Optional.ofNullable(newlyLoaded) + .orElse(Collections.emptyList()) + .forEach(item -> { + String code = normalizeValue(item.getCode()); + if (code != null) { + categoryCacheByCode.put(code, item); + mapping.put(code, item); + } + }); + missing.removeIf(mapping::containsKey); + if (!missing.isEmpty()) { + List preview = missing.stream().limit(10).toList(); + log.warn("当前批次存在 {} 个分类未同步，示例: {}", missing.size(), preview); + } + } + return mapping; + } + + private Map upsertMaterials(List batch, + MasterDataSyncReport report, + Map materialCacheById, + Map materialCacheByCode) { + Set sourceIds = batch.stream() + .map(MdmMaterialViewDO::getCodeId) + .filter(Objects::nonNull) + .filter(id -> id > 0) + .collect(Collectors.toCollection(LinkedHashSet::new)); + if (sourceIds.isEmpty()) { + log.warn("本次批次缺少有效 CODEID，无法同步物料主数据"); + return Collections.emptyMap(); + } + Map existingById = sourceIds.stream() + .map(materialCacheById::get) + .filter(Objects::nonNull) + .collect(Collectors.toMap(MaterialInfomationDO::getId, Function.identity(), (a, b) -> a, LinkedHashMap::new)); + Set missingIds = new LinkedHashSet<>(sourceIds); + missingIds.removeIf(existingById::containsKey); + if (!missingIds.isEmpty()) { + List fetched = materialInfomationMapper.selectBatchIds(missingIds); + Optional.ofNullable(fetched) + .orElse(Collections.emptyList()) + .forEach(item -> { + if (item.getId() != null) { + existingById.put(item.getId(), item); + materialCacheById.put(item.getId(), item); + String normalizedCode = normalizeValue(item.getCode()); + if (normalizedCode != null) { + materialCacheByCode.put(normalizedCode, item); + } + } + }); + } + Map mapping = new LinkedHashMap<>(); + List pendingMaterialInserts = new ArrayList<>(); + for (MdmMaterialViewDO item : batch) { + Long codeId = item.getCodeId(); + if (codeId == null || codeId <= 0) { + log.warn("物料 {} 缺少主键 CODEID，已跳过", item.getMaterialCode()); + continue; + } + String code = normalizeValue(item.getMaterialCode()); + if (code == null) { + continue; + } + MaterialInfomationDO current = existingById.get(codeId); + String name = normalizeValue(item.getMaterialName()); + if (current == null) { + MaterialInfomationDO created = MaterialInfomationDO.builder() + .id(codeId) + .code(code) + .name(StrUtil.emptyIfNull(name)) + .remark(null) + .build(); + pendingMaterialInserts.add(created); + existingById.put(codeId, created); + materialCacheById.put(codeId, created); + materialCacheByCode.put(code, created); + mapping.put(code, created); + report.incrementInsertedMaterials(); + continue; + } + + boolean needUpdate = false; + MaterialInfomationDO update = null; + if (!StrUtil.equals(current.getCode(), code)) { + current.setCode(code); + needUpdate = true; + } + if (name != null && !StrUtil.equals(current.getName(), name)) { + current.setName(name); + needUpdate = true; + } + if (needUpdate) { + update = new MaterialInfomationDO(); + update.setId(current.getId()); + update.setCode(current.getCode()); + update.setName(current.getName()); + materialInfomationMapper.updateById(update); + report.incrementUpdatedMaterials(); + } + mapping.put(code, current); + materialCacheByCode.put(code, current); + } + if (CollUtil.isNotEmpty(pendingMaterialInserts)) { + materialInfomationMapper.insertBatch(pendingMaterialInserts); + } + return mapping; + } + + private void bindMaterialClasses(List batch, + Map materials, + Map categoryClasses, + MasterDataSyncReport report) { + if (materials.isEmpty() || categoryClasses.isEmpty()) { + return; + } + List infoIds = batch.stream() + .map(MdmMaterialViewDO::getMaterialCode) + .map(this::normalizeValue) + .map(materials::get) + .filter(Objects::nonNull) + .map(MaterialInfomationDO::getId) + .filter(Objects::nonNull) + .distinct() + .collect(Collectors.toList()); + Map existingRelations = materialHasClassesMapper.selectByInfoIds(infoIds) + .stream() + .collect(Collectors.toMap(MaterialHasClassesDO::getInfomationId, Function.identity(), (a, b) -> a)); + List relationsToInsert = new ArrayList<>(); + for (MdmMaterialViewDO item : batch) { + String materialCode = normalizeValue(item.getMaterialCode()); + MaterialInfomationDO info = materials.get(materialCode); + if (info == null) { + continue; + } + String categoryCode = normalizeValue(item.getCategoryCode()); + MaterialClassesDO cls = categoryClasses.get(categoryCode); + if (cls == null) { + continue; + } + MaterialHasClassesDO relation = existingRelations.get(info.getId()); + if (relation == null) { + MaterialHasClassesDO created = MaterialHasClassesDO.builder() + .infomationId(info.getId()) + .classesId(cls.getId()) + .build(); + relationsToInsert.add(created); + existingRelations.put(info.getId(), created); + report.incrementCreatedClassRelations(); + } else if (!Objects.equals(relation.getClassesId(), cls.getId())) { + MaterialHasClassesDO update = new MaterialHasClassesDO(); + update.setId(relation.getId()); + update.setClassesId(cls.getId()); + materialHasClassesMapper.updateById(update); + relation.setClassesId(cls.getId()); + report.incrementUpdatedClassRelations(); + } + } + if (CollUtil.isNotEmpty(relationsToInsert)) { + materialHasClassesMapper.insertBatch(relationsToInsert); + } + } + + private void upsertMaterialProperties(List batch, + Map materials, + Map propertyDefinitions, + MasterDataSyncReport report) { + if (materials.isEmpty() || propertyDefinitions.isEmpty()) { + return; + } + List infoIds = batch.stream() + .map(MdmMaterialViewDO::getMaterialCode) + .map(this::normalizeValue) + .map(materials::get) + .filter(Objects::nonNull) + .map(MaterialInfomationDO::getId) + .filter(Objects::nonNull) + .distinct() + .collect(Collectors.toList()); + List propertyIds = propertyDefinitions.values().stream() + .map(MaterialPropertiesDO::getId) + .filter(Objects::nonNull) + .distinct() + .collect(Collectors.toList()); + Map existing = materialHasPropertiesMapper + .selectByInfoIdsAndPropertyIds(infoIds, propertyIds) + .stream() + .collect(Collectors.toMap(item -> buildRelationKey(item.getInfomationId(), item.getPropertiesId()), + Function.identity(), (a, b) -> a)); + List propertyRelationsToInsert = new ArrayList<>(); + for (MdmMaterialViewDO item : batch) { + String materialCode = normalizeValue(item.getMaterialCode()); + MaterialInfomationDO info = materials.get(materialCode); + if (info == null || info.getId() == null) { + continue; + } + for (Map.Entry entry : propertyDefinitions.entrySet()) { + MaterialPropertiesDO property = entry.getValue(); + if (property == null || property.getId() == null) { + continue; + } + String value = normalizeValue(entry.getKey().extractValue(item)); + if (value == null) { + continue; + } + String key = buildRelationKey(info.getId(), property.getId()); + MaterialHasPropertiesDO relation = existing.get(key); + if (relation == null) { + MaterialHasPropertiesDO created = MaterialHasPropertiesDO.builder() + .infomationId(info.getId()) + .propertiesId(property.getId()) + .value(value) + .sort(entry.getKey().getSort()) + .isKey(0) + .isMetering(0) + .build(); + propertyRelationsToInsert.add(created); + existing.put(key, created); + report.incrementInsertedPropertyValues(); + } else if (!StrUtil.equals(relation.getValue(), value) + || !Objects.equals(relation.getSort(), entry.getKey().getSort())) { + MaterialHasPropertiesDO update = new MaterialHasPropertiesDO(); + update.setId(relation.getId()); + update.setValue(value); + update.setSort(entry.getKey().getSort()); + materialHasPropertiesMapper.updateById(update); + relation.setValue(value); + relation.setSort(entry.getKey().getSort()); + report.incrementUpdatedPropertyValues(); + } + } + } + if (CollUtil.isNotEmpty(propertyRelationsToInsert)) { + materialHasPropertiesMapper.insertBatch(propertyRelationsToInsert); + } + } + + private String buildRelationKey(Long infoId, Long propertyId) { + return infoId + ":" + propertyId; + } + + private String normalizeValue(String value) { + return StrUtil.isBlank(value) ? null : StrUtil.trim(value); + } +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/dto/MasterDataCategorySyncReport.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/dto/MasterDataCategorySyncReport.java new file mode 100644 index 00000000..50985d12 --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/dto/MasterDataCategorySyncReport.java @@ -0,0 +1,59 @@ +package com.zt.plat.module.base.service.masterdatasync.dto; + +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * 物料分类同步执行结果。 + */ +@Data +public class MasterDataCategorySyncReport { + + private long processedCategories; + private long insertedCategories; + private long updatedCategories; + private long missingParentReferences; + private boolean success; + private String message; + private LocalDateTime startedAt; + private LocalDateTime finishedAt; + + public static MasterDataCategorySyncReport start() { + MasterDataCategorySyncReport report = new MasterDataCategorySyncReport(); + report.setStartedAt(LocalDateTime.now()); + report.setSuccess(false); + report.setMessage("执行中"); + return report; + } + + public void markSuccess() { + this.success = true; + this.message = "执行成功"; + } + + public void markFailure(String message) { + this.success = false; + this.message = message; + } + + public void finish() { + this.finishedAt = LocalDateTime.now(); + } + + public void incrementProcessed(long delta) { + this.processedCategories += delta; + } + + public void incrementInserted() { + this.insertedCategories++; + } + + public void incrementUpdated() { + this.updatedCategories++; + } + + public void incrementMissingParent() { + this.missingParentReferences++; + } +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/dto/MasterDataSyncCommand.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/dto/MasterDataSyncCommand.java new file mode 100644 index 00000000..cbc32777 --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/dto/MasterDataSyncCommand.java @@ -0,0 +1,33 @@ +package com.zt.plat.module.base.service.masterdatasync.dto; + +import lombok.Data; + +import java.time.LocalDateTime; +import java.util.List; + +/** + * 主数据同步的入参模型。 + */ +@Data +public class MasterDataSyncCommand { + + /** + * 增量同步的起始时间，空值表示全量。 + */ + private LocalDateTime since; + + /** + * 单批拉取的记录数，空值取配置默认值。 + */ + private Integer batchSize; + + /** + * 需要限定同步范围的物料编码集合。 + */ + private List materialCodes; + + /** + * 本次同步允许处理的最大记录数，空值表示不限制。 + */ + private Long recordLimit; +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/dto/MasterDataSyncReport.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/dto/MasterDataSyncReport.java new file mode 100644 index 00000000..aa5300ca --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/dto/MasterDataSyncReport.java @@ -0,0 +1,104 @@ +package com.zt.plat.module.base.service.masterdatasync.dto; + +import lombok.Data; + +import java.time.LocalDateTime; +import java.util.Collections; +import java.util.List; + +/** + * 主数据同步的执行结果统计。 + */ +@Data +public class MasterDataSyncReport { + + private long processedRecords; + private long insertedMaterials; + private long updatedMaterials; + private long insertedClasses; + private long updatedClasses; + private long createdClassRelations; + private long updatedClassRelations; + private long insertedPropertyDefinitions; + private long updatedPropertyDefinitions; + private long insertedPropertyValues; + private long updatedPropertyValues; + private LocalDateTime startedAt; + private LocalDateTime finishedAt; + private boolean success; + private String message; + private Integer batchSize; + private LocalDateTime since; + private List materialCodes = Collections.emptyList(); + private Long recordLimit; + + public static MasterDataSyncReport start(MasterDataSyncCommand command) { + MasterDataSyncReport report = new MasterDataSyncReport(); + report.setStartedAt(LocalDateTime.now()); + report.setBatchSize(command.getBatchSize()); + report.setSince(command.getSince()); + report.setMaterialCodes(command.getMaterialCodes()); + report.setRecordLimit(command.getRecordLimit()); + report.setSuccess(false); + report.setMessage("执行中"); + return report; + } + + public void finish() { + this.finishedAt = LocalDateTime.now(); + } + + public void markSuccess() { + this.success = true; + this.message = "执行成功"; + } + + public void markFailure(String message) { + this.success = false; + this.message = message; + } + + public void incrementProcessedRecords(long delta) { + this.processedRecords += delta; + } + + public void incrementInsertedMaterials() { + this.insertedMaterials++; + } + + public void incrementUpdatedMaterials() { + this.updatedMaterials++; + } + + public void incrementInsertedClasses() { + this.insertedClasses++; + } + + public void incrementUpdatedClasses() { + this.updatedClasses++; + } + + public void incrementCreatedClassRelations() { + this.createdClassRelations++; + } + + public void incrementUpdatedClassRelations() { + this.updatedClassRelations++; + } + + public void incrementInsertedPropertyDefinitions() { + this.insertedPropertyDefinitions++; + } + + public void incrementUpdatedPropertyDefinitions() { + this.updatedPropertyDefinitions++; + } + + public void incrementInsertedPropertyValues() { + this.insertedPropertyValues++; + } + + public void incrementUpdatedPropertyValues() { + this.updatedPropertyValues++; + } +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/support/MasterDataPropertyDefinition.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/support/MasterDataPropertyDefinition.java new file mode 100644 index 00000000..33545eef --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/support/MasterDataPropertyDefinition.java @@ -0,0 +1,52 @@ +package com.zt.plat.module.base.service.masterdatasync.support; + +import com.zt.plat.module.base.dal.dataobject.masterdata.MdmMaterialViewDO; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.time.format.DateTimeFormatter; +import java.util.function.Function; + +/** + * 源视图字段与物料属性定义之间的标准映射关系。 + */ +@Getter +@RequiredArgsConstructor +public enum MasterDataPropertyDefinition { + + BASE_UNIT_CODE("MTRL_BASE_UNIT_CODE", "基本计量单位编码", 50L, MdmMaterialViewDO::getBaseUnitCode, "STRING"), + BASE_UNIT_NAME("MTRL_BASE_UNIT_NAME", "基本计量单位描述", 60L, MdmMaterialViewDO::getBaseUnitName, "STRING"), + SHORT_DESCRIPTION("MTRL_SHORT_DESC", "短描述", 70L, MdmMaterialViewDO::getShortDescription, "STRING"), + LONG_DESCRIPTION("MTRL_LONG_DESC", "长描述", 80L, MdmMaterialViewDO::getLongDescription, "STRING"), + CHALCO_CODE("MTRL_CHALCO_CODE", "中铝编码", 90L, MdmMaterialViewDO::getChalcoCode, "STRING"), + SPECIFICATION("MTRL_SPECIFICATION", "规格", 100L, MdmMaterialViewDO::getSpecification, "STRING"), + MODEL("MTRL_MODEL", "型号", 110L, MdmMaterialViewDO::getModel, "STRING"), + TEXTURE("MTRL_TEXTURE", "材质", 120L, MdmMaterialViewDO::getTexture, "STRING"), + DRAWING_NUMBER("MTRL_DRAWING_NUMBER", "图号", 130L, MdmMaterialViewDO::getDrawingNumber, "STRING"), + ORDER_NUMBER("MTRL_ORDER_NUMBER", "订货号", 140L, MdmMaterialViewDO::getOrderNumber, "STRING"), + OTHER_PARAMETERS("MTRL_OTHER_PARAMETERS", "其它参数", 150L, MdmMaterialViewDO::getOtherParameters, "STRING"), + EQUIPMENT_CATEGORY("MTRL_EQUIPMENT_CATEGORY", "设备类别", 160L, MdmMaterialViewDO::getEquipmentCategory, "STRING"), + MANUFACTURER("MTRL_MANUFACTURER", "主机生产商", 170L, MdmMaterialViewDO::getManufacturer, "STRING"), + SOURCE("MTRL_SOURCE", "来源", 180L, MdmMaterialViewDO::getSource, "STRING"), + RECORD_TIME("MTRL_RECORD_TIME", "记录时间", 190L, + source -> source.getRecordTime() == null ? null : source.getRecordTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME), + "DATETIME"); + + private final String code; + private final String displayName; + private final long sort; + private final Function extractor; + private final String dataType; + + public String extractValue(MdmMaterialViewDO source) { + if (source == null) { + return null; + } + String value = extractor.apply(source); + if (value == null) { + return null; + } + value = value.trim(); + return value.isEmpty() ? null : value; + } +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/support/MasterDataSyncNotificationPayload.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/support/MasterDataSyncNotificationPayload.java new file mode 100644 index 00000000..995ef43b --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/support/MasterDataSyncNotificationPayload.java @@ -0,0 +1,16 @@ +package com.zt.plat.module.base.service.masterdatasync.support; + +import com.zt.plat.module.base.service.masterdatasync.dto.MasterDataSyncReport; +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * 同步结束后推送给下游监听方的 JSON 载荷。 + */ +@Data +@AllArgsConstructor +public class MasterDataSyncNotificationPayload { + + private MasterDataSyncReport report; + private String errorMessage; +} diff --git a/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/support/MasterDataSyncNotifier.java b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/support/MasterDataSyncNotifier.java new file mode 100644 index 00000000..32037ed1 --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/java/com/zt/plat/module/base/service/masterdatasync/support/MasterDataSyncNotifier.java @@ -0,0 +1,61 @@ +package com.zt.plat.module.base.service.masterdatasync.support; + +import cn.hutool.core.util.StrUtil; +import com.zt.plat.framework.common.util.json.JsonUtils; +import com.zt.plat.module.base.framework.sync.config.MasterDataSyncProperties; +import com.zt.plat.module.base.service.masterdatasync.dto.MasterDataSyncReport; +import lombok.extern.slf4j.Slf4j; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; + +import java.io.IOException; + +/** + * 在同步结束后发送回调通知。 + */ +@Slf4j +@Component +public class MasterDataSyncNotifier { + + private static final MediaType JSON = MediaType.get("application/json; charset=UTF-8"); + + private final OkHttpClient httpClient; + private final MasterDataSyncProperties properties; + + public MasterDataSyncNotifier(@Qualifier("masterDataSyncOkHttpClient") OkHttpClient httpClient, + MasterDataSyncProperties properties) { + this.httpClient = httpClient; + this.properties = properties; + } + + public void dispatch(MasterDataSyncReport report, Throwable failure) { + if (report == null || StrUtil.isBlank(properties.getCallbackUrl())) { + return; + } + if (report.isSuccess() && !properties.isNotifyOnSuccess()) { + return; + } + if (!report.isSuccess() && !properties.isNotifyOnFailure()) { + return; + } + MasterDataSyncNotificationPayload payload = new MasterDataSyncNotificationPayload(report, + failure == null ? null : failure.getMessage()); + RequestBody body = RequestBody.create(JSON, JsonUtils.toJsonString(payload)); + Request.Builder builder = new Request.Builder() + .url(properties.getCallbackUrl()) + .post(body); + properties.getCallbackHeaders().forEach(builder::addHeader); + try (Response response = httpClient.newCall(builder.build()).execute()) { + if (!response.isSuccessful()) { + log.warn("主数据同步回调响应非成功状态: {}", response.code()); + } + } catch (IOException ex) { + log.warn("调用主数据同步回调接口失败", ex); + } + } +} diff --git a/zt-module-base/zt-module-base-server/src/main/resources/mapper/masterdata/MdmMaterialCategoryMapper.xml b/zt-module-base/zt-module-base-server/src/main/resources/mapper/masterdata/MdmMaterialCategoryMapper.xml new file mode 100644 index 00000000..426986bb --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/resources/mapper/masterdata/MdmMaterialCategoryMapper.xml @@ -0,0 +1,17 @@ + + + + + + + diff --git a/zt-module-base/zt-module-base-server/src/main/resources/mapper/masterdata/MdmMaterialViewMapper.xml b/zt-module-base/zt-module-base-server/src/main/resources/mapper/masterdata/MdmMaterialViewMapper.xml new file mode 100644 index 00000000..0e3c31e2 --- /dev/null +++ b/zt-module-base/zt-module-base-server/src/main/resources/mapper/masterdata/MdmMaterialViewMapper.xml @@ -0,0 +1,71 @@ + + + + + + a.codeid AS code_id, + a.code AS material_code, + a.categorycode AS category_code, + a.categoryname AS category_name, + a.desc1 AS material_name, + a.desc6 AS base_unit_code, + a.desc19 AS base_unit_name, + a.descshort AS short_description, + a.desclong AS long_description, + a.desc86 AS chalco_code, + b.pur_class AS major_class_code, + c.desc1 AS major_class_name, + a.desc9 AS specification, + a.desc10 AS model, + a.desc11 AS texture, + a.desc12 AS drawing_number, + a.desc13 AS order_number, + a.desc14 AS other_parameters, + a.desc15 AS equipment_category, + a.desc16 AS manufacturer, + a.desc17 AS source, + a.recordtime AS record_time + + + + mdm_wlzsj_code a + INNER JOIN z_pur_class b ON a.codeid = b.codeid + INNER JOIN mdm_wlfl_code c ON b.pur_class = c.code + + + + + + diff --git a/zt-module-base/主数据同步说明.md b/zt-module-base/主数据同步说明.md new file mode 100644 index 00000000..33881d50 --- /dev/null +++ b/zt-module-base/主数据同步说明.md @@ -0,0 +1,140 @@ +# 主数据同步功能说明 + + +## 功能概述 + +主数据同步由“分类同步 + 物料同步”两条独立链路组成： + +1. 分类同步：从 `mdm_wlfl_code` 全量拉取 `01 / 0101 / 010101` 结构的编码，生成 BSE 三层分类。 +2. 物料同步：从 `mdm_material_view` 拉取物料主数据，依赖已存在的分类信息完成入库。 +3. 物料同步会维护物料与分类绑定关系，并将非主表字段落入属性表。 +4. 同步结束后可选地向外部回调推送执行报告。 + +## 同步入口 + +### 分类同步接口 + +- **接口地址**：`POST /base/master-data-sync/categories` +- **权限点**：`base:master-data-sync:categories` +- **请求体**：无（全量同步） +- **返回体**：`MasterDataCategorySyncReport` + +字段说明： + +- `processedCategories`：本次遍历的分类数量。 +- `insertedCategories` / `updatedCategories`：新增或更新的分类记录数。 +- `missingParentReferences`：外部数据缺失父级时的降级次数（会挂到根节点）。 +- `success`、`message`、`startedAt`、`finishedAt`：同物料同步。 + +> ⚠️ 分类同步需先于物料同步执行，确保 BSE 中已存在三层分类结构。 + +### 物料同步接口 + +- **接口地址**：`POST /base/master-data-sync/execute` +- **权限点**：`base:master-data-sync:execute` +- **请求体**： + + | 字段 | 类型 | 说明 | + | --- | --- | --- | + | `since` | `yyyy-MM-dd HH:mm:ss` | （可选）增量同步起始时间，空值表示全量。 | + | `batchSize` | `Integer` | （可选）单批拉取条数，未填写则使用配置默认值。 | + | `materialCodes` | `List` | （可选）限定仅同步指定物料编码，留空同步全部。 | + | `recordLimit` | `Long` | （可选）本次同步允许处理的最大记录数，留空则不做总量限制。 | + +**返回体**为 `MasterDataSyncReport`，涵盖以下关键指标： + +- `processedRecords`：拉取并处理的记录数。 +- `insertedMaterials` / `updatedMaterials`：新增或更新的物料数量。 +- `createdClassRelations` / `updatedClassRelations`：物料与分类的关系处理次数。 +- `insertedPropertyDefinitions` / `updatedPropertyDefinitions`：属性定义的补齐情况。 +- `insertedPropertyValues` / `updatedPropertyValues`：属性值的写入情况。 +- `success` + `message`：执行状态及提示语。 +- `startedAt` / `finishedAt`：执行起止时间。 +- `recordLimit`：若请求指定了总量限制，会在报告中回显该值。 + +## 数据处理策略 + +1. **属性字典**： + - 程序会检查 `MasterDataPropertyDefinition` 枚举列出的全部属性（例如类别编码、规格、材质等）。 + - 若属性定义不存在即创建，存在但名称/排序有差异会自动更新。 +2. **分类管理**： + - 通过分类同步接口预先写入 `01/0101/010101` 三层数据，物料同步阶段仅检查并使用现有分类，如缺失则记录日志并跳过绑定。 +3. **物料主体**： + - 按 `material_code` 进行 upsert，保留基础字段（物料编码、通用信息等）。 +4. **分类绑定**： + - 若物料已有绑定但分类变化，则更新关系；否则创建新绑定。 +5. **属性值写入**： + - 对比新旧值，仅在有变化或缺失时写入，避免无效更新。 + +## 配置项 + +在 `application-*.yml` 中通过 `base.master-data-sync` 节点进行控制： + +```yaml +base: + master-data-sync: + enabled: true # 是否开放同步入口 + batch-size: 500 # 默认批量大小 + source-datasource-name: mdm # 指向外部 MDM 的动态数据源 + notify-on-success: true # 成功时是否回调 + notify-on-failure: true # 失败时是否回调 + callback-url: http://example/callback + callback-headers: + X-TOKEN: xxx + http: + connect-timeout: 5s + read-timeout: 30s + write-timeout: 30s +``` + +> ⚠️ 请确保在 `datasource.dynamic` 中配置名为 `mdm`（或 `source-datasource-name` 指定值）的数据源，并指向外部 MDM 数据库。 + +## 数据源说明 + +- `master`：默认业务库（BSE 表所在库），所有增删改依旧走 `master` 数据源，这部分无需额外配置。 +- `mdm`：主数据来源库（外部 MySQL）。`MdmMaterialViewMapper`、`MdmMaterialCategoryMapper` 均通过 `@DS("mdm")` 读取该库。 +- `base.master-data-sync.source-datasource-name`：若外部库名称不是 `mdm`，可在此覆盖，示例： + + ```yaml + base: + master-data-sync: + source-datasource-name: mdm + datasource: + dynamic: + datasource: + master: { ...本地库配置... } + mdm: { url: jdbc:mysql://mdm-host/mdm, username: xxx, password: yyy } + ``` + +> ✅ 强调：主数据读取始终走 `mdm`（或你在 `source-datasource-name` 中声明的别名），不要与默认 `master` 数据源混淆。 + +## 回调机制 + +- 当配置了 `callbackUrl` 时，系统会在同步结束后推送 JSON 结果。 +- 推送内容为： + +```json +{ + "report": { ...MasterDataSyncReport... }, + "errorMessage": "可选的错误提示" +} +``` + +- 仅成功/失败分别受 `notifyOnSuccess`、`notifyOnFailure` 控制。 +- `callbackHeaders` 支持附加认证信息，如 Token、租户标识等。 + +## 常见问题排查 + +| 问题 | 检查点 | +| --- | --- | +| 数据源连接失败 | 确认 `mdm` 数据源配置、账号权限、网络可达性。 | +| 同步后未触发回调 | 检查 `callbackUrl`、`notify-on-*` 配置，以及目标地址是否可达。 | +| 属性值未生效 | 查看 `MasterDataPropertyDefinition` 是否包含该字段，或字段值是否为空/仅有空格。 | +| 只需同步单个物料 | 调用接口时在 `materialCodes` 中传入目标编码即可。 | + +## 建议的使用流程 + +1. 配置并验证外部数据源与 `base.master-data-sync` 参数。 +2. 先执行“分类同步接口”，确认三层分类齐全。 +3. 再执行“物料同步接口”，并确认报表数据、回调通知。 +4. 线上场景可定期以 `since` 参数驱动增量物料同步，也可按需指定 `materialCodes` 触发刷新；分类若有改动，可重新全量执行一次分类同步。