diff --git a/backend/src/main/java/com/qqchen/deploy/backend/deploy/api/RepositoryManagerApiController.java b/backend/src/main/java/com/qqchen/deploy/backend/deploy/api/RepositoryManagerApiController.java index 5e690521..dc2b7355 100644 --- a/backend/src/main/java/com/qqchen/deploy/backend/deploy/api/RepositoryManagerApiController.java +++ b/backend/src/main/java/com/qqchen/deploy/backend/deploy/api/RepositoryManagerApiController.java @@ -20,14 +20,14 @@ import org.springframework.web.bind.annotation.*; public class RepositoryManagerApiController { @Resource - private IRepositoryManagerService gitManagerService; + private IRepositoryManagerService repositoryManagerService; @Operation(summary = "同步Git仓库组", description = "同步指定外部系统的所有仓库组") @PostMapping("/{externalSystemId}/sync-groups") public Response syncGroups( @Parameter(description = "外部系统ID", required = true) @PathVariable Long externalSystemId ) { - gitManagerService.syncGroups(externalSystemId); + repositoryManagerService.syncGroups(externalSystemId); return Response.success(); } @@ -36,7 +36,7 @@ public class RepositoryManagerApiController { public Response syncProjects( @Parameter(description = "外部系统ID", required = true) @PathVariable Long externalSystemId ) { - gitManagerService.syncProjects(externalSystemId); + repositoryManagerService.syncProjects(externalSystemId); return Response.success(); } @@ -45,7 +45,7 @@ public class RepositoryManagerApiController { public Response syncBranches( @Parameter(description = "外部系统ID", required = true) @PathVariable Long externalSystemId ) { - gitManagerService.syncBranches(externalSystemId); + repositoryManagerService.syncBranches(externalSystemId); return Response.success(); } @@ -54,6 +54,6 @@ public class RepositoryManagerApiController { public Response instance( @Parameter(description = "外部系统ID", required = true) @PathVariable Long externalSystemId ) { - return Response.success(gitManagerService.instance(externalSystemId)); + return Response.success(repositoryManagerService.instance(externalSystemId)); } } \ No newline at end of file diff --git a/backend/src/main/java/com/qqchen/deploy/backend/deploy/entity/RepositoryProject.java b/backend/src/main/java/com/qqchen/deploy/backend/deploy/entity/RepositoryProject.java index 4399c66f..2668cc15 100644 --- a/backend/src/main/java/com/qqchen/deploy/backend/deploy/entity/RepositoryProject.java +++ b/backend/src/main/java/com/qqchen/deploy/backend/deploy/entity/RepositoryProject.java @@ -16,7 +16,6 @@ import java.time.LocalDateTime; @LogicDelete public class RepositoryProject extends Entity { - @Column(nullable = false) private String name; @@ -30,8 +29,8 @@ public class RepositoryProject extends Entity { @Column(name = "group_id") private Long groupId; - @Column(name = "is_default_branch") - private String isDefaultBranch; + @Column(name = "default_branch") + private String defaultBranch; @Column(name = "web_url") private String webUrl; @@ -45,6 +44,15 @@ public class RepositoryProject extends Entity { @Column(name = "last_activity_at") private LocalDateTime lastActivityAt; + @Column(name = "name_with_namespace") + private String nameWithNamespace; + + @Column(name = "path_with_namespace") + private String pathWithNamespace; + + @Column(name = "created_at") + private LocalDateTime createdAt; + @Column(name = "external_system_id") private Long externalSystemId; diff --git a/backend/src/main/java/com/qqchen/deploy/backend/deploy/integration/IGitServiceIntegration.java b/backend/src/main/java/com/qqchen/deploy/backend/deploy/integration/IGitServiceIntegration.java index 92c4fe68..731f36bc 100644 --- a/backend/src/main/java/com/qqchen/deploy/backend/deploy/integration/IGitServiceIntegration.java +++ b/backend/src/main/java/com/qqchen/deploy/backend/deploy/integration/IGitServiceIntegration.java @@ -30,6 +30,15 @@ public interface IGitServiceIntegration extends IExternalSystemIntegration { */ List projects(ExternalSystem system); + /** + * 获取指定组下的所有项目 + * + * @param system 外部系统配置 + * @param groupId 组ID + * @return 项目列表 + */ + List projectsByGroup(ExternalSystem system, Long groupId); + /** * 获取指定项目的所有分支 * diff --git a/backend/src/main/java/com/qqchen/deploy/backend/deploy/integration/impl/GitServiceIntegrationImpl.java b/backend/src/main/java/com/qqchen/deploy/backend/deploy/integration/impl/GitServiceIntegrationImpl.java index 724874d0..d3c8690d 100644 --- a/backend/src/main/java/com/qqchen/deploy/backend/deploy/integration/impl/GitServiceIntegrationImpl.java +++ b/backend/src/main/java/com/qqchen/deploy/backend/deploy/integration/impl/GitServiceIntegrationImpl.java @@ -71,7 +71,7 @@ public class GitServiceIntegrationImpl implements IGitServiceIntegration { @Override public List projects(ExternalSystem system) { try { - String url = String.format("%s/api/v4/projects?per_page=100", system.getUrl()); + String url = String.format("%s/api/v4/projects", system.getUrl()); HttpHeaders headers = createHeaders(system); HttpEntity entity = new HttpEntity<>(headers); @@ -89,6 +89,27 @@ public class GitServiceIntegrationImpl implements IGitServiceIntegration { } } + @Override + public List projectsByGroup(ExternalSystem system, Long groupId) { + try { + String url = String.format("%s/api/v4/groups/%d/projects?per_page=100", system.getUrl(), groupId); + HttpHeaders headers = createHeaders(system); + HttpEntity entity = new HttpEntity<>(headers); + + ResponseEntity> response = restTemplate.exchange( + url, + HttpMethod.GET, + entity, + new ParameterizedTypeReference<>() {} + ); + + return response.getBody() != null ? response.getBody() : Collections.emptyList(); + } catch (Exception e) { + log.error("Failed to fetch git projects for system: {} and group: {}", system.getName(), groupId, e); + return Collections.emptyList(); + } + } + @Override public List branches(ExternalSystem system, Long projectId) { try { diff --git a/backend/src/main/java/com/qqchen/deploy/backend/deploy/integration/response/GitProjectResponse.java b/backend/src/main/java/com/qqchen/deploy/backend/deploy/integration/response/GitProjectResponse.java index dd7beb9f..93b33330 100644 --- a/backend/src/main/java/com/qqchen/deploy/backend/deploy/integration/response/GitProjectResponse.java +++ b/backend/src/main/java/com/qqchen/deploy/backend/deploy/integration/response/GitProjectResponse.java @@ -1,8 +1,11 @@ package com.qqchen.deploy.backend.deploy.integration.response; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; /** * Git项目响应对象 @@ -20,13 +23,81 @@ public class GitProjectResponse { private String visibility; + @JsonProperty("default_branch") private String defaultBranch; + @JsonProperty("web_url") private String webUrl; - private String sshUrl; + @JsonProperty("ssh_url_to_repo") + private String sshUrlToRepo; - private String httpUrl; + @JsonProperty("http_url_to_repo") + private String httpUrlToRepo; + @JsonProperty("last_activity_at") private LocalDateTime lastActivityAt; + + @JsonProperty("name_with_namespace") + private String nameWithNamespace; + + @JsonProperty("path_with_namespace") + private String pathWithNamespace; + + @JsonProperty("created_at") + private LocalDateTime createdAt; + + @JsonProperty("tag_list") + private List tagList; + + @JsonProperty("readme_url") + private String readmeUrl; + + @JsonProperty("avatar_url") + private String avatarUrl; + + @JsonProperty("forks_count") + private Integer forksCount; + + @JsonProperty("star_count") + private Integer starCount; + + @JsonProperty("packages_enabled") + private Boolean packagesEnabled; + + @JsonProperty("empty_repo") + private Boolean emptyRepo; + + @JsonProperty("archived") + private Boolean archived; + + @JsonProperty("container_registry_enabled") + private Boolean containerRegistryEnabled; + + @JsonProperty("container_expiration_policy") + private Map containerExpirationPolicy; + + @JsonProperty("resolve_outdated_diff_discussions") + private Boolean resolveOutdatedDiffDiscussions; + + private Namespace namespace; + + @JsonProperty("_links") + private Map links; + + @Data + public static class Namespace { + private Long id; + private String name; + private String path; + private String kind; + @JsonProperty("full_path") + private String fullPath; + @JsonProperty("parent_id") + private Long parentId; + @JsonProperty("avatar_url") + private String avatarUrl; + @JsonProperty("web_url") + private String webUrl; + } } \ No newline at end of file diff --git a/backend/src/main/java/com/qqchen/deploy/backend/deploy/service/impl/RepositoryBranchServiceImpl.java b/backend/src/main/java/com/qqchen/deploy/backend/deploy/service/impl/RepositoryBranchServiceImpl.java index 8204e9c0..04698860 100644 --- a/backend/src/main/java/com/qqchen/deploy/backend/deploy/service/impl/RepositoryBranchServiceImpl.java +++ b/backend/src/main/java/com/qqchen/deploy/backend/deploy/service/impl/RepositoryBranchServiceImpl.java @@ -60,13 +60,11 @@ public class RepositoryBranchServiceImpl extends BaseServiceImpl new BusinessException(ResponseCode.EXTERNAL_SYSTEM_NOT_FOUND)); + ExternalSystem externalSystem = externalSystemRepository.findById(externalSystemId).orElseThrow(() -> new BusinessException(ResponseCode.EXTERNAL_SYSTEM_NOT_FOUND)); // 3. 获取所有项目 List projects = repositoryProjectRepository.findByExternalSystemIdAndDeletedFalse(externalSystemId); diff --git a/backend/src/main/java/com/qqchen/deploy/backend/deploy/service/impl/RepositoryProjectServiceImpl.java b/backend/src/main/java/com/qqchen/deploy/backend/deploy/service/impl/RepositoryProjectServiceImpl.java index 7794c22a..778ee8a4 100644 --- a/backend/src/main/java/com/qqchen/deploy/backend/deploy/service/impl/RepositoryProjectServiceImpl.java +++ b/backend/src/main/java/com/qqchen/deploy/backend/deploy/service/impl/RepositoryProjectServiceImpl.java @@ -24,6 +24,8 @@ import java.util.List; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.HashSet; +import java.util.Set; /** * Git仓库项目服务实现 @@ -54,47 +56,99 @@ public class RepositoryProjectServiceImpl extends BaseServiceImpl new BusinessException(ResponseCode.EXTERNAL_SYSTEM_NOT_FOUND)); + log.info("Start syncing projects for external system: {} (ID: {})", externalSystem.getName(), externalSystemId); - // 2. 获取远程仓库项目信息 - List remoteProjects = gitServiceIntegration.projects(externalSystem); - if (remoteProjects.isEmpty()) { - log.info("No projects found in remote git system: {}", externalSystem.getName()); + // 2. 获取所有组 + List groups = repositoryGroupRepository.findByExternalSystemIdAndDeletedFalse(externalSystemId); + if (groups.isEmpty()) { + log.info("No groups found for external system: {}", externalSystem.getName()); return 0; } + log.info("Found {} groups to sync", groups.size()); - // 3. 获取本地已存在的项目 - List existingProjects = repositoryProjectRepository.findByExternalSystemIdAndDeletedFalse(externalSystemId); - Map existingProjectMap = existingProjects.stream() - .collect(Collectors.toMap(RepositoryProject::getProjectId, Function.identity())); + // 3. 获取现有项目映射（包括已删除的） + Map existingProjects = repositoryProjectRepository + .findByExternalSystemId(externalSystemId) + .stream() + .collect(Collectors.toMap( + RepositoryProject::getProjectId, + Function.identity(), + (existing, replacement) -> { + log.warn("Duplicate project found with ID: {}, path: {}", + existing.getProjectId(), existing.getPathWithNamespace()); + return existing; + } + )); - // 4. 更新或创建项目 - List projectsToSave = remoteProjects.stream() - .map(remoteProject -> { - RepositoryProject project = existingProjectMap.getOrDefault(remoteProject.getId(), new RepositoryProject()); - project.setExternalSystemId(externalSystemId); - project.setProjectId(remoteProject.getId()); - project.setName(remoteProject.getName()); - project.setPath(remoteProject.getPath()); - project.setDescription(remoteProject.getDescription()); - project.setVisibility(remoteProject.getVisibility()); - project.setIsDefaultBranch(remoteProject.getDefaultBranch()); - project.setWebUrl(remoteProject.getWebUrl()); - project.setSshUrl(remoteProject.getSshUrl()); - project.setHttpUrl(remoteProject.getHttpUrl()); - project.setLastActivityAt(remoteProject.getLastActivityAt()); - return project; - }) - .collect(Collectors.toList()); + // 4. 用于跟踪已处理的项目ID + Set processedProjectIds = new HashSet<>(); + int totalCount = 0; - List savedProjects = repositoryProjectRepository.saveAll(projectsToSave); - - log.info("Successfully synchronized {} projects for external system: {}", - savedProjects.size(), externalSystem.getName()); - return savedProjects.size(); - + // 5. 遍历每个组，同步项目 + for (RepositoryGroup group : groups) { + List projectResponses = gitServiceIntegration.projectsByGroup(externalSystem, group.getGroupId()); + log.info("Processing group: {} (ID: {}), found {} projects", + group.getName(), group.getGroupId(), projectResponses.size()); + + for (GitProjectResponse projectResponse : projectResponses) { + if (processedProjectIds.contains(projectResponse.getId())) { + log.info("Project already processed: {} (ID: {}) in group: {}", + projectResponse.getPathWithNamespace(), projectResponse.getId(), group.getName()); + continue; + } + processedProjectIds.add(projectResponse.getId()); + + RepositoryProject project = existingProjects.get(projectResponse.getId()); + if (project == null) { + project = new RepositoryProject(); + project.setExternalSystemId(externalSystemId); + project.setProjectId(projectResponse.getId()); + log.info("Creating new project: {} (ID: {})", + projectResponse.getPathWithNamespace(), projectResponse.getId()); + totalCount++; + } else { + log.debug("Updating existing project: {} (ID: {})", + projectResponse.getPathWithNamespace(), projectResponse.getId()); + } + + project.setDeleted(false); + project.setName(projectResponse.getName()); + project.setPath(projectResponse.getPath()); + project.setDescription(projectResponse.getDescription()); + project.setVisibility(projectResponse.getVisibility()); + project.setGroupId(group.getId()); + project.setDefaultBranch(projectResponse.getDefaultBranch()); + project.setWebUrl(projectResponse.getWebUrl()); + project.setSshUrl(projectResponse.getSshUrlToRepo()); + project.setHttpUrl(projectResponse.getHttpUrlToRepo()); + project.setLastActivityAt(projectResponse.getLastActivityAt()); + project.setNameWithNamespace(projectResponse.getNameWithNamespace()); + project.setPathWithNamespace(projectResponse.getPathWithNamespace()); + project.setCreatedAt(projectResponse.getCreatedAt()); + + repositoryProjectRepository.save(project); + existingProjects.remove(projectResponse.getId()); + } + } + + // 6. 删除不存在的项目 + if (!existingProjects.isEmpty()) { + List toDeleteIds = existingProjects.values().stream() + .filter(p -> !p.getDeleted()) + .map(RepositoryProject::getId) + .collect(Collectors.toList()); + if (!toDeleteIds.isEmpty()) { + log.info("Deleting {} projects that no longer exist", toDeleteIds.size()); + repositoryProjectRepository.deleteAllById(toDeleteIds); + } + } + + log.info("Successfully synced projects. Added {} new projects, processed {} total projects", + totalCount, processedProjectIds.size()); + return totalCount; } catch (Exception e) { log.error("Failed to sync repository projects for external system: {}", externalSystemId, e); - throw new BusinessException(ResponseCode.REPOSITORY_SYNC_FAILED); + throw new BusinessException(ResponseCode.REPOSITORY_PROJECT_SYNC_FAILED); } } diff --git a/backend/src/main/resources/db/migration/V1.0.0__init_schema.sql b/backend/src/main/resources/db/migration/V1.0.0__init_schema.sql index 95b443a2..57d50b43 100644 --- a/backend/src/main/resources/db/migration/V1.0.0__init_schema.sql +++ b/backend/src/main/resources/db/migration/V1.0.0__init_schema.sql @@ -305,32 +305,34 @@ CREATE TABLE deploy_repo_group -- 代码仓库项目表 CREATE TABLE deploy_repo_project ( - -- 基础字段 - id BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键ID', - create_by VARCHAR(100) NULL COMMENT '创建人', - create_time DATETIME(6) NULL COMMENT '创建时间', - update_by VARCHAR(100) NULL COMMENT '更新人', - update_time DATETIME(6) NULL COMMENT '更新时间', - version INT NOT NULL DEFAULT 1 COMMENT '版本号', - deleted BIT NOT NULL DEFAULT 0 COMMENT '是否删除：0-未删除，1-已删除', + id BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键ID', + create_by VARCHAR(255) NULL COMMENT '创建人', + create_time DATETIME(6) NULL COMMENT '创建时间', + deleted BIT NOT NULL DEFAULT 0 COMMENT '是否删除（0：未删除，1：已删除）', + update_by VARCHAR(255) NULL COMMENT '更新人', + update_time DATETIME(6) NULL COMMENT '更新时间', + version INT NOT NULL DEFAULT 0 COMMENT '乐观锁版本号', - -- 业务字段 - name VARCHAR(100) NOT NULL COMMENT '项目名称', - description VARCHAR(500) NULL COMMENT '项目描述', - project_id BIGINT NOT NULL COMMENT '外部系统中的项目ID', - group_id BIGINT NULL COMMENT '所属仓库组ID', - path VARCHAR(200) NOT NULL COMMENT '项目路径', - external_system_id BIGINT NOT NULL COMMENT '外部系统ID', - is_default_branch VARCHAR(100) NULL COMMENT '默认分支', - is_enabled BIT NOT NULL DEFAULT 1 COMMENT '是否启用：0-禁用，1-启用', - visibility ENUM('private', 'internal', 'public') NOT NULL DEFAULT 'private' COMMENT '可见性：private-私有，internal-内部，public-公开', - http_url VARCHAR(255) NULL COMMENT 'HTTP克隆地址', - ssh_url VARCHAR(255) NULL COMMENT 'SSH克隆地址', - web_url VARCHAR(255) NULL COMMENT '网页URL', - last_activity_at DATETIME(6) NULL COMMENT '最后活动时间', - sort INT DEFAULT 0 COMMENT '排序号' + name VARCHAR(255) NOT NULL COMMENT '项目名称', + path VARCHAR(255) NOT NULL COMMENT '项目路径', + description TEXT NULL COMMENT '项目描述', + visibility VARCHAR(50) NULL COMMENT '可见性', + group_id BIGINT NULL COMMENT '所属组ID', + default_branch VARCHAR(100) NULL COMMENT '默认分支', + web_url VARCHAR(500) NULL COMMENT 'Web URL', + ssh_url VARCHAR(500) NULL COMMENT 'SSH URL', + http_url VARCHAR(500) NULL COMMENT 'HTTP URL', + last_activity_at DATETIME(6) NULL COMMENT '最后活动时间', + name_with_namespace VARCHAR(500) NULL COMMENT '带命名空间的名称', + path_with_namespace VARCHAR(500) NULL COMMENT '带命名空间的路径', + created_at DATETIME(6) NULL COMMENT '创建时间', + external_system_id BIGINT NOT NULL COMMENT '外部系统ID', + project_id BIGINT NOT NULL COMMENT '项目ID', -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='代码仓库项目表'; + CONSTRAINT UK_repo_project_external_system_id_project_id UNIQUE (external_system_id, project_id), + CONSTRAINT FK_repo_project_group FOREIGN KEY (group_id) REFERENCES deploy_repo_group (id), + CONSTRAINT FK_repo_project_external_system FOREIGN KEY (external_system_id) REFERENCES sys_external_system (id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='Git仓库项目表'; -- 代码仓库分支表 CREATE TABLE deploy_repo_branch