|
108 | 108 | import java.util.Collections; |
109 | 109 | import java.util.HashMap; |
110 | 110 | import java.util.HashSet; |
| 111 | +import java.util.LinkedHashMap; |
111 | 112 | import java.util.LinkedList; |
112 | 113 | import java.util.List; |
113 | 114 | import java.util.Map; |
@@ -147,9 +148,6 @@ public class PartitionManager { |
147 | 148 | public static final String CONSENSUS_WRITE_ERROR = |
148 | 149 | "Failed in the write API executing the consensus layer due to: "; |
149 | 150 |
|
150 | | - private static final long REGION_GROUP_VISIBILITY_TIMEOUT_MS = 10_000L; |
151 | | - private static final long REGION_GROUP_VISIBILITY_CHECK_INTERVAL_MS = 20L; |
152 | | - |
153 | 151 | // Monitor for leadership change |
154 | 152 | private final Object scheduleMonitor = new Object(); |
155 | 153 |
|
@@ -718,75 +716,12 @@ private TSStatus generateAndAllocateRegionGroups( |
718 | 716 | getLoadManager().allocateRegionGroups(allotmentMap, consensusGroupType); |
719 | 717 | LOGGER.info(ManagerMessages.CREATEREGIONGROUPS_STARTING_TO_CREATE_THE_FOLLOWING_REGIONGROUPS); |
720 | 718 | createRegionGroupsPlan.planLog(LOGGER); |
721 | | - final TSStatus createStatus = |
722 | | - getProcedureManager().createRegionGroups(consensusGroupType, createRegionGroupsPlan); |
723 | | - if (createStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { |
724 | | - return createStatus; |
725 | | - } |
726 | | - return waitForRegionGroupsVisible(createRegionGroupsPlan, consensusGroupType); |
| 719 | + return getProcedureManager().createRegionGroups(consensusGroupType, createRegionGroupsPlan); |
727 | 720 | } else { |
728 | 721 | return RpcUtils.SUCCESS_STATUS; |
729 | 722 | } |
730 | 723 | } |
731 | 724 |
|
732 | | - private TSStatus waitForRegionGroupsVisible( |
733 | | - final CreateRegionGroupsPlan createRegionGroupsPlan, |
734 | | - final TConsensusGroupType consensusGroupType) { |
735 | | - final Map<String, Set<TConsensusGroupId>> expectedRegionGroups = new HashMap<>(); |
736 | | - createRegionGroupsPlan |
737 | | - .getRegionGroupMap() |
738 | | - .forEach( |
739 | | - (database, regionReplicaSets) -> { |
740 | | - final Set<TConsensusGroupId> regionGroupIds = |
741 | | - regionReplicaSets.stream() |
742 | | - .map(TRegionReplicaSet::getRegionId) |
743 | | - .filter(regionGroupId -> consensusGroupType.equals(regionGroupId.getType())) |
744 | | - .collect(Collectors.toSet()); |
745 | | - if (!regionGroupIds.isEmpty()) { |
746 | | - expectedRegionGroups.put(database, regionGroupIds); |
747 | | - } |
748 | | - }); |
749 | | - |
750 | | - final long startTime = System.currentTimeMillis(); |
751 | | - while (System.currentTimeMillis() - startTime <= REGION_GROUP_VISIBILITY_TIMEOUT_MS) { |
752 | | - if (areRegionGroupsVisible(expectedRegionGroups, consensusGroupType)) { |
753 | | - return RpcUtils.SUCCESS_STATUS; |
754 | | - } |
755 | | - try { |
756 | | - TimeUnit.MILLISECONDS.sleep(REGION_GROUP_VISIBILITY_CHECK_INTERVAL_MS); |
757 | | - } catch (final InterruptedException e) { |
758 | | - Thread.currentThread().interrupt(); |
759 | | - return new TSStatus(TSStatusCode.CREATE_REGION_ERROR.getStatusCode()) |
760 | | - .setMessage( |
761 | | - String.format( |
762 | | - "Interrupted while waiting for created %s RegionGroups %s to become visible in PartitionInfo.", |
763 | | - consensusGroupType, expectedRegionGroups)); |
764 | | - } |
765 | | - } |
766 | | - |
767 | | - final String message = |
768 | | - String.format( |
769 | | - "Created %s RegionGroups %s are not visible in PartitionInfo within %d ms.", |
770 | | - consensusGroupType, expectedRegionGroups, REGION_GROUP_VISIBILITY_TIMEOUT_MS); |
771 | | - LOGGER.warn(message); |
772 | | - return new TSStatus(TSStatusCode.CREATE_REGION_ERROR.getStatusCode()).setMessage(message); |
773 | | - } |
774 | | - |
775 | | - private boolean areRegionGroupsVisible( |
776 | | - final Map<String, Set<TConsensusGroupId>> expectedRegionGroups, |
777 | | - final TConsensusGroupType consensusGroupType) { |
778 | | - for (final Map.Entry<String, Set<TConsensusGroupId>> entry : expectedRegionGroups.entrySet()) { |
779 | | - final Set<TConsensusGroupId> visibleRegionGroups = |
780 | | - partitionInfo.getRegionGroupSlotsCounter(entry.getKey(), consensusGroupType).stream() |
781 | | - .map(Pair::getRight) |
782 | | - .collect(Collectors.toSet()); |
783 | | - if (!visibleRegionGroups.containsAll(entry.getValue())) { |
784 | | - return false; |
785 | | - } |
786 | | - } |
787 | | - return true; |
788 | | - } |
789 | | - |
790 | 725 | /** |
791 | 726 | * Only leader use this interface. Checks whether the specified DataPartition has a successor and |
792 | 727 | * returns if it does. |
@@ -1059,6 +994,23 @@ public List<Pair<Long, TConsensusGroupId>> getSortedRegionGroupSlotsCounter( |
1059 | 994 | } |
1060 | 995 |
|
1061 | 996 | if (result.isEmpty()) { |
| 997 | + // Diagnostic for the intermittent "no available RegionGroup" CI failures: dump every |
| 998 | + // RegionGroup visible in PartitionInfo for this Database together with its LoadCache status. |
| 999 | + // This pinpoints whether PartitionInfo simply has no RegionGroup yet (newly created |
| 1000 | + // RegionGroup not exposed) or it has some but all of them are currently Disabled. |
| 1001 | + // Only logged on the failure path right before throwing, so it never floods the log. |
| 1002 | + final Map<TConsensusGroupId, RegionGroupStatus> visibleRegionGroupStatusMap = |
| 1003 | + new LinkedHashMap<>(); |
| 1004 | + regionGroupSlotsCounter.forEach( |
| 1005 | + slotsCounter -> |
| 1006 | + visibleRegionGroupStatusMap.put( |
| 1007 | + slotsCounter.getRight(), |
| 1008 | + getLoadManager().getRegionGroupStatus(slotsCounter.getRight()))); |
| 1009 | + LOGGER.warn( |
| 1010 | + "No available {} RegionGroup for Database: {}. RegionGroups visible in PartitionInfo and their LoadCache status: {}", |
| 1011 | + type, |
| 1012 | + database, |
| 1013 | + visibleRegionGroupStatusMap); |
1062 | 1014 | throw new NoAvailableRegionGroupException(type, Collections.singletonList(database)); |
1063 | 1015 | } |
1064 | 1016 |
|
|
0 commit comments