Re-range clone sequences to new group id during promotion

src/backend/distributed/metadata/metadata_sync.c

@@ -115,6 +115,7 @@ static bool ShouldSyncTableMetadataInternal(bool hashDistributed,
 static bool SyncNodeMetadataSnapshotToNode(WorkerNode *workerNode, bool raiseOnError);
 static void DropMetadataSnapshotOnNode(WorkerNode *workerNode);
 static void FetchSequenceState(Oid sequenceId, int64 *lastValue, bool *isCalled);
+static void AppendSequenceRangeAdjustCommand(Oid sequenceId, List **commandList);
 static char * CreateSequenceDependencyCommand(Oid relationId, Oid sequenceId,
 											  char *columnName);
 static GrantStmt * GenerateGrantStmtForRights(ObjectType objectType,
@@ -2008,31 +2009,81 @@ IdentitySequenceDependencyCommandList(Oid targetRelationId)
 			missingOk
 			);
 
-		char *qualifiedSequenceName = generate_qualified_relation_name(sequenceId);
+		AppendSequenceRangeAdjustCommand(sequenceId, &commandList);
+	}
 
-		/* prevent concurrent updates to the sequence until the end of the transaction */
-		LockRelationOid(sequenceId, RowExclusiveLock);
+	relation_close(relation, NoLock);
 
-		int64 lastValue = 0;
-		bool isCalled = false;
-		FetchSequenceState(sequenceId, &lastValue, &isCalled);
+	return commandList;
+}
 
-		StringInfo stringInfo = makeStringInfo();
-		appendStringInfo(stringInfo,
-						 WORKER_ADJUST_IDENTITY_COLUMN_SEQ_SETTINGS,
-						 quote_literal_cstr(qualifiedSequenceName),
-						 lastValue, isCalled ? "true" : "false");
 
-		commandList = lappend(commandList,
-							  makeTableDDLCommandString(stringInfo->data));
-	}
+/*
+ * DependentSequenceRangeAdjustCommandList generates a list of commands to
+ * execute WORKER_ADJUST_IDENTITY_COLUMN_SEQ_SETTINGS for each nextval/serial
+ * sequence that the given relation depends on, so that those sequences' min/max
+ * values are re-ranged to the target node's local group-id window (producing
+ * globally-unique values across groups).
+ *
+ * This mirrors IdentitySequenceDependencyCommandList but covers sequence-backed
+ * default columns (serial, bigint DEFAULT nextval(...)) rather than identity
+ * columns. The worker side of the reused UDF runs AlterSequenceMinMax(), the
+ * same routine the classical activation path invokes via
+ * AdjustDependentSeqRangesOnLocalWorker(). It is used by the clone-promotion
+ * path to re-range the promoted clone's sequences to its newly-allocated group
+ * id, which the physical-replica clone would otherwise inherit from its source
+ * primary.
+ */
+List *
+DependentSequenceRangeAdjustCommandList(Oid relationId)
+{
+	List *commandList = NIL;
 
-	relation_close(relation, NoLock);
+	List *seqInfoList = NIL;
+	GetDependentSequencesWithRelation(relationId, &seqInfoList, 0, DEPENDENCY_AUTO);
+
+	SequenceInfo *seqInfo = NULL;
+	foreach_declared_ptr(seqInfo, seqInfoList)
+	{
+		AppendSequenceRangeAdjustCommand(seqInfo->sequenceOid, &commandList);
+	}
 
 	return commandList;
 }
 
 
+/*
+ * AppendSequenceRangeAdjustCommand appends a WORKER_ADJUST_IDENTITY_COLUMN_SEQ_SETTINGS
+ * command for the given sequence to commandList. The command re-ranges the
+ * sequence's min/max values to the target node's local group-id window on
+ * workers (and sets last_value/is_called on the coordinator).
+ *
+ * It is shared by IdentitySequenceDependencyCommandList and
+ * DependentSequenceRangeAdjustCommandList so the two stay in sync.
+ */
+static void
+AppendSequenceRangeAdjustCommand(Oid sequenceId, List **commandList)
+{
+	char *qualifiedSequenceName = generate_qualified_relation_name(sequenceId);
+
+	/* prevent concurrent updates to the sequence until the end of the transaction */
+	LockRelationOid(sequenceId, RowExclusiveLock);
+
+	int64 lastValue = 0;
+	bool isCalled = false;
+	FetchSequenceState(sequenceId, &lastValue, &isCalled);
+
+	StringInfo stringInfo = makeStringInfo();
+	appendStringInfo(stringInfo,
+					 WORKER_ADJUST_IDENTITY_COLUMN_SEQ_SETTINGS,
+					 quote_literal_cstr(qualifiedSequenceName),
+					 lastValue, isCalled ? "true" : "false");
+
+	*commandList = lappend(*commandList,
+						   makeTableDDLCommandString(stringInfo->data));
+}
+
+
 /*
  * FetchSequenceState fetches the last_value and is_called for the sequence with
  * given oid.

src/backend/distributed/operations/node_promotion.c

@@ -5,17 +5,20 @@
 
 #include "distributed/argutils.h"
 #include "distributed/clonenode_utils.h"
+#include "distributed/coordinator_protocol.h"
 #include "distributed/listutils.h"
 #include "distributed/metadata_cache.h"
 #include "distributed/metadata_sync.h"
 #include "distributed/remote_commands.h"
 #include "distributed/shard_rebalancer.h"
+#include "distributed/worker_transaction.h"
 
 
 static void BlockAllWritesToWorkerNode(WorkerNode *workerNode);
 static bool GetNodeIsInRecoveryStatus(WorkerNode *workerNode);
 static void PromoteCloneNode(WorkerNode *cloneWorkerNode);
 static void EnsureSingleNodePromotion(WorkerNode *primaryNode);
+static void AdjustCloneSequenceRangesForNewGroup(WorkerNode *cloneNode);
 
 PG_FUNCTION_INFO_V1(citus_promote_clone_and_rebalance);
 
@@ -197,6 +200,16 @@ citus_promote_clone_and_rebalance(PG_FUNCTION_ARGS)
 	 */
 	SyncNodeMetadataToNodes();
 
+	/*
+	 * Re-range the promoted clone's distributed-table sequences to its new
+	 * group-id window. The clone is a physical replica, so its sequence objects
+	 * still carry the source primary's (groupId << 48) range; the classical
+	 * activation path does the equivalent re-ranging per group id. This must run
+	 * after SyncNodeMetadataToNodes() so the clone's local group id is already
+	 * corrected and visible over the reused metadata connection.
+	 */
+	AdjustCloneSequenceRangesForNewGroup(cloneNode);
+
 	/* Step 5: Split Shards Between Primary and Clone */
 	SplitShardsBetweenPrimaryAndClone(primaryNode, cloneNode, PG_GETARG_NAME_OR_NULL(1))
 	;
@@ -422,3 +435,90 @@ EnsureSingleNodePromotion(WorkerNode *primaryNode)
 		;
 	}
 }
+
+
+/*
+ * AdjustCloneSequenceRangesForNewGroup re-ranges the promoted clone's
+ * sequence-backed columns for the same table surface that classical
+ * add/activate-node flow syncs to metadata workers (i.e. tables where
+ * ShouldSyncTableMetadata(relationId) is true).
+ *
+ * A clone is a physical streaming replica of its source primary, so its
+ * sequence objects are byte-for-byte copies that still carve out the source
+ * primary's (groupId << 48) value window. The classical activation path
+ * re-ranges sequences per group id (via AlterSequenceMinMax) so each group emits
+ * globally-unique values; the clone path must do the same once its group id is
+ * corrected.
+ *
+ * The command list is built on the coordinator (reusing the existing per-table
+ * sequence command builders) and sent to the clone over the metadata connection
+ * via SendMetadataCommandListToWorkerListInCoordinatedTransaction. That path
+ * reuses the same connection SyncNodeMetadataToNodes() used to update
+ * pg_dist_local_group, so the corrected local group id is visible when
+ * AlterSequenceMinMax() runs on the clone. Therefore this MUST be called after
+ * SyncNodeMetadataToNodes().
+ */
+static void
+AdjustCloneSequenceRangesForNewGroup(WorkerNode *cloneNode)
+{
+	List *ddlCommandList = NIL;
+
+	List *citusTableIdList = AllCitusTableIds();
+	Oid relationId = InvalidOid;
+	foreach_declared_oid(relationId, citusTableIdList)
+	{
+		if (!ShouldSyncTableMetadata(relationId))
+		{
+			continue;
+		}
+
+		ddlCommandList = list_concat(ddlCommandList,
+									 DependentSequenceRangeAdjustCommandList(relationId));
+		ddlCommandList = list_concat(ddlCommandList,
+									 IdentitySequenceDependencyCommandList(relationId));
+	}
+
+	if (ddlCommandList == NIL)
+	{
+		/* no sequence-backed Citus-table columns to re-range */
+		return;
+	}
+
+	/*
+	 * SendMetadataCommandListToWorkerListInCoordinatedTransaction expects plain
+	 * command strings, so unwrap each TableDDLCommand.
+	 */
+	List *commandList = NIL;
+	TableDDLCommand *ddlCommand = NULL;
+	foreach_declared_ptr(ddlCommand, ddlCommandList)
+	{
+		commandList = lappend(commandList, GetTableDDLCommand(ddlCommand));
+	}
+
+	/*
+	 * Re-fetch the clone node so its hasMetadata/metadataSynced flags reflect
+	 * the post-activation, post-sync catalog state. The cloneNode passed in was
+	 * loaded at the start of promotion (before ActivateCloneNodeAsPrimary and
+	 * SyncNodeMetadataToNodes), so its in-memory metadata flags are still those
+	 * of an inactive clone and would trip the metadata-node sanity checks in the
+	 * send path.
+	 */
+	WorkerNode *freshCloneNode = FindNodeAnyClusterByNodeId(cloneNode->nodeId);
+	if (freshCloneNode == NULL)
+	{
+		ereport(ERROR, (errmsg("could not find promoted clone node with ID %d "
+							   "while re-ranging its sequences",
+							   cloneNode->nodeId)));
+	}
+
+	SendMetadataCommandListToWorkerListInCoordinatedTransaction(
+		list_make1(freshCloneNode),
+		CurrentUserName(),
+		commandList);
+
+	ereport(NOTICE, (errmsg(
+						 "re-ranged %d sequence(s) on promoted clone %s:%d (ID %d) for new group %d",
+						 list_length(commandList), freshCloneNode->workerName,
+						 freshCloneNode->workerPort, freshCloneNode->nodeId,
+						 freshCloneNode->groupId)));
+}

src/include/distributed/metadata_sync.h

@@ -127,6 +127,7 @@ extern void SignalMetadataSyncDaemon(Oid database, int sig);
 extern bool ShouldInitiateMetadataSync(bool *lockFailure);
 extern List * SequenceDependencyCommandList(Oid relationId);
 extern List * IdentitySequenceDependencyCommandList(Oid targetRelationId);
+extern List * DependentSequenceRangeAdjustCommandList(Oid relationId);
 
 extern List * DDLCommandsForSequence(Oid sequenceOid, char *ownerName);
 extern List * GetSequencesFromAttrDef(Oid attrdefOid);

src/test/regress/citus_tests/run_test.py

@@ -120,6 +120,9 @@ def extra_tests(self):
     "multi_add_node_from_backup_sync_replica": TestDeps(
         None, repeatable=False, worker_count=5
     ),
+    "multi_add_node_from_backup_sequences": TestDeps(
+        None, ["multi_add_node_from_backup_negative"], worker_count=5, repeatable=False
+    ),
     "single_node": TestDeps(None, ["multi_test_helpers"]),
     "single_node_truncate": TestDeps(None),
     "multi_explain": TestDeps(

src/test/regress/expected/multi_add_node_from_backup.out

@@ -243,6 +243,7 @@ NOTICE:  Clone localhost:xxxxx is now caught up with primary localhost:xxxxx.
 NOTICE:  Attempting to promote clone localhost:xxxxx via pg_promote().
 NOTICE:  Clone node localhost:xxxxx (ID 3) has been successfully promoted.
 NOTICE:  Updating metadata for promoted clone localhost:xxxxx (ID 3)
+NOTICE:  re-ranged 6 sequence(s) on promoted clone localhost:xxxxx (ID 3) for new group 3
 NOTICE:  adjusting shard placements for primary localhost:xxxxx and clone localhost:xxxxx
 NOTICE:  processing 4 shards for primary node GroupID 1
 LOG:  inserting DELETE shard record for shard public.table1_colg2_102020 from clone node GroupID 3

src/test/regress/expected/multi_add_node_from_backup_negative.out

@@ -119,6 +119,7 @@ NOTICE:  Clone localhost:xxxxx is now caught up with primary localhost:xxxxx.
 NOTICE:  Attempting to promote clone localhost:xxxxx via pg_promote().
 NOTICE:  Clone node localhost:xxxxx (ID 4) has been successfully promoted.
 NOTICE:  Updating metadata for promoted clone localhost:xxxxx (ID 4)
+NOTICE:  re-ranged 6 sequence(s) on promoted clone localhost:xxxxx (ID 4) for new group 4
 ERROR:  could not find rebalance strategy with name invalid_strategy
 SELECT * from pg_dist_node ORDER by nodeid;
  nodeid | groupid | nodename  | nodeport | noderack | hasmetadata | isactive |  noderole   | nodecluster | metadatasynced | shouldhaveshards | nodeisclone | nodeprimarynodeid
@@ -228,6 +229,7 @@ NOTICE:  Clone localhost:xxxxx is now caught up with primary localhost:xxxxx.
 NOTICE:  Attempting to promote clone localhost:xxxxx via pg_promote().
 NOTICE:  Clone node localhost:xxxxx (ID 6) has been successfully promoted.
 NOTICE:  Updating metadata for promoted clone localhost:xxxxx (ID 6)
+NOTICE:  re-ranged 6 sequence(s) on promoted clone localhost:xxxxx (ID 6) for new group 6
 NOTICE:  adjusting shard placements for primary localhost:xxxxx and clone localhost:xxxxx
 NOTICE:  processing 13 shards for primary node GroupID 5
 LOG:  inserting DELETE shard record for shard public.backup_test2_102091 from clone node GroupID 6