From 7b9a274cd7bb84735a2c1692c2ac4038d4dbcea5 Mon Sep 17 00:00:00 2001 From: SaketaChalamchala Date: Wed, 24 Jun 2026 16:08:22 -0700 Subject: [PATCH 1/3] HDDS-15387. Added SnapshotDiffValueParser to parse only required fiedlds for diff generation from Key/DirectoryInfo. --- .../om/snapshot/SnapshotDiffValueParser.java | 338 ++++++++++++++++++ .../snapshot/TestSnapshotDiffValueParser.java | 191 ++++++++++ 2 files changed, 529 insertions(+) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffValueParser.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffValueParser.java diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffValueParser.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffValueParser.java new file mode 100644 index 000000000000..58d193ba17c6 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffValueParser.java @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyRequest.HSYNC_FIELD_NUMBER; + +import com.google.protobuf.ByteString; +import com.google.protobuf.CodedInputStream; +import com.google.protobuf.WireFormat; +import java.io.IOException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.KeyValue; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DirectoryInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; + +/** + * Parses snapshot diff values without full deserialization. + */ +public final class SnapshotDiffValueParser { + private static final int INT_BYTES = 4; + private static final int LONG_BYTES = 8; + private static final String DIGEST_ALGORITHM = "SHA-256"; + + private SnapshotDiffValueParser() { + } + + public static ParsedRequiredInfo parseKeyInfoRequiredFields(byte[] value, boolean includeUpdateId) + throws IOException { + CodedInputStream input = CodedInputStream.newInstance(value); + long updateId = 0L; + long objectId = 0L; + long parentId = 0L; + boolean hasUpdateId = false; + String keyName = null; + + int tag; + while ((tag = input.readTag()) != 0) { + int fieldNumber = WireFormat.getTagFieldNumber(tag); + switch (fieldNumber) { + case KeyInfo.KEYNAME_FIELD_NUMBER: + keyName = input.readString(); + break; + case KeyInfo.OBJECTID_FIELD_NUMBER: + objectId = input.readUInt64(); + break; + case KeyInfo.UPDATEID_FIELD_NUMBER: + if (includeUpdateId) { + updateId = input.readUInt64(); + hasUpdateId = true; + } else { + input.skipField(tag); + } + break; + case KeyInfo.PARENTID_FIELD_NUMBER: + parentId = input.readUInt64(); + break; + default: + input.skipField(tag); + break; + } + } + + return new ParsedRequiredInfo(updateId, hasUpdateId, objectId, parentId, keyName); + } + + public static byte[] computeKeyInfoCompareSignature(byte[] value) throws IOException { + CodedInputStream input = CodedInputStream.newInstance(value); + MessageDigest digest = newDigest(); + boolean hasHsyncMetadata = false; + int keyLocationListCount = 0; + ByteString latestKeyLocationList = null; + + int tag; + while ((tag = input.readTag()) != 0) { + int fieldNumber = WireFormat.getTagFieldNumber(tag); + switch (fieldNumber) { + case KeyInfo.DATASIZE_FIELD_NUMBER: + updateDigestWithLong(digest, fieldNumber, input.readUInt64()); + break; + case KeyInfo.KEYLOCATIONLIST_FIELD_NUMBER: + latestKeyLocationList = input.readBytes(); + keyLocationListCount++; + break; + case KeyInfo.METADATA_FIELD_NUMBER: + MetadataEntry metadataEntry = parseMetadataEntry(input.readBytes().toByteArray()); + if (metadataEntry != null) { + updateDigestWithRawBytes(digest, fieldNumber, metadataEntry.getDigest()); + if (metadataEntry.hasHsync()) { + hasHsyncMetadata = true; + } + } + break; + case KeyInfo.FILECHECKSUM_FIELD_NUMBER: + case KeyInfo.ACLS_FIELD_NUMBER: + case KeyInfo.TAGS_FIELD_NUMBER: + updateDigestWithBytes(digest, fieldNumber, input.readBytes()); + break; + default: + input.skipField(tag); + break; + } + } + + if (latestKeyLocationList != null) { + updateDigestWithBytes(digest, KeyInfo.KEYLOCATIONLIST_FIELD_NUMBER, latestKeyLocationList); + } + updateDigestWithBoolean(digest, HSYNC_FIELD_NUMBER, hasHsyncMetadata); + updateDigestWithInt(digest, keyLocationListCount); + + return digest.digest(); + } + + public static ParsedRequiredInfo parseDirectoryInfoRequiredFields(byte[] value, boolean includeUpdateId) + throws IOException { + CodedInputStream input = CodedInputStream.newInstance(value); + long updateId = 0L; + long objectId = 0L; + long parentId = 0L; + boolean hasUpdateId = false; + String name = null; + + int tag; + while ((tag = input.readTag()) != 0) { + int fieldNumber = WireFormat.getTagFieldNumber(tag); + switch (fieldNumber) { + case DirectoryInfo.NAME_FIELD_NUMBER: + name = input.readString(); + break; + case DirectoryInfo.OBJECTID_FIELD_NUMBER: + objectId = input.readUInt64(); + break; + case DirectoryInfo.UPDATEID_FIELD_NUMBER: + if (includeUpdateId) { + updateId = input.readUInt64(); + hasUpdateId = true; + } else { + input.skipField(tag); + } + break; + case DirectoryInfo.PARENTID_FIELD_NUMBER: + parentId = input.readUInt64(); + break; + default: + input.skipField(tag); + break; + } + } + + return new ParsedRequiredInfo(updateId, hasUpdateId, objectId, parentId, name); + } + + public static byte[] computeDirectoryInfoCompareSignature(byte[] value) throws IOException { + CodedInputStream input = CodedInputStream.newInstance(value); + MessageDigest digest = newDigest(); + + int tag; + while ((tag = input.readTag()) != 0) { + int fieldNumber = WireFormat.getTagFieldNumber(tag); + switch (fieldNumber) { + case DirectoryInfo.METADATA_FIELD_NUMBER: + MetadataEntry metadataEntry = parseMetadataEntry(input.readBytes().toByteArray()); + if (metadataEntry != null) { + updateDigestWithRawBytes(digest, fieldNumber, metadataEntry.getDigest()); + } + break; + case DirectoryInfo.ACLS_FIELD_NUMBER: + updateDigestWithBytes(digest, fieldNumber, input.readBytes()); + break; + default: + input.skipField(tag); + break; + } + } + + return digest.digest(); + } + + private static void updateDigestWithLong(MessageDigest digest, int fieldNumber, long value) { + updateDigestWithInt(digest, fieldNumber); + byte[] buffer = new byte[LONG_BYTES]; + for (int i = LONG_BYTES - 1; i >= 0; i--) { + buffer[i] = (byte) (value & 0xFFL); + value >>>= 8; + } + digest.update(buffer); + } + + private static void updateDigestWithBytes(MessageDigest digest, int fieldNumber, ByteString value) { + updateDigestWithInt(digest, fieldNumber); + updateDigestWithInt(digest, value.size()); + digest.update(value.toByteArray()); + } + + private static void updateTaggedString(MessageDigest digest, int fieldNumber, String value) { + updateDigestWithInt(digest, fieldNumber); + if (value == null) { + updateDigestWithInt(digest, 0); + return; + } + byte[] bytes = value.getBytes(java.nio.charset.StandardCharsets.UTF_8); + updateDigestWithInt(digest, bytes.length); + digest.update(bytes); + } + + private static void updateDigestWithBoolean(MessageDigest digest, int fieldNumber, boolean value) { + updateDigestWithInt(digest, fieldNumber); + updateDigestWithInt(digest, value ? 1 : 0); + } + + private static void updateDigestWithRawBytes(MessageDigest digest, int fieldNumber, byte[] value) { + updateDigestWithInt(digest, fieldNumber); + updateDigestWithInt(digest, value.length); + digest.update(value); + } + + private static void updateDigestWithInt(MessageDigest digest, int value) { + byte[] buffer = new byte[INT_BYTES]; + for (int i = INT_BYTES - 1; i >= 0; i--) { + buffer[i] = (byte) (value & 0xFF); + value >>>= 8; + } + digest.update(buffer); + } + + private static MessageDigest newDigest() { + try { + return MessageDigest.getInstance(DIGEST_ALGORITHM); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is not available", e); + } + } + + private static MetadataEntry parseMetadataEntry(byte[] metadataBytes) throws IOException { + if (metadataBytes == null || metadataBytes.length == 0) { + return null; + } + CodedInputStream input = CodedInputStream.newInstance(metadataBytes); + String key = null; + String value = null; + while (!input.isAtEnd()) { + int tag = input.readTag(); + if (tag == 0) { + break; + } + int fieldNumber = WireFormat.getTagFieldNumber(tag); + switch (fieldNumber) { + case KeyValue.KEY_FIELD_NUMBER: + key = input.readString(); + break; + case KeyValue.VALUE_FIELD_NUMBER: + value = input.readString(); + break; + default: + input.skipField(tag); + break; + } + } + MessageDigest entryDigest = newDigest(); + updateTaggedString(entryDigest, KeyValue.KEY_FIELD_NUMBER, key); + updateTaggedString(entryDigest, KeyValue.VALUE_FIELD_NUMBER, value); + return new MetadataEntry(entryDigest.digest(), OzoneConsts.HSYNC_CLIENT_ID.equals(key)); + } + + private static final class MetadataEntry { + private final byte[] digest; + private final boolean hasHsync; + + private MetadataEntry(byte[] digest, boolean hasHsync) { + this.digest = digest; + this.hasHsync = hasHsync; + } + + private byte[] getDigest() { + return digest; + } + + private boolean hasHsync() { + return hasHsync; + } + } + + /** + * Parsed fields shared by key and directory entries. + * Holds IDs and name with optional updateID when requested. + */ + public static final class ParsedRequiredInfo { + private final long updateId; + private final boolean hasUpdateId; + private final long objectId; + private final long parentId; + private final String name; + + private ParsedRequiredInfo(long updateId, boolean hasUpdateId, long objectId, long parentId, String name) { + this.updateId = updateId; + this.hasUpdateId = hasUpdateId; + this.objectId = objectId; + this.parentId = parentId; + this.name = name; + } + + public long getUpdateId() { + return updateId; + } + + public boolean hasUpdateId() { + return hasUpdateId; + } + + public long getObjectId() { + return objectId; + } + + public long getParentId() { + return parentId; + } + + public String getName() { + return name; + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffValueParser.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffValueParser.java new file mode 100644 index 000000000000..292227cb1b55 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffValueParser.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum; +import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; +import org.apache.hadoop.io.MD5Hash; +import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.junit.jupiter.api.Test; + +class TestSnapshotDiffValueParser { + private static final String VOLUME = "volume"; + private static final String BUCKET = "bucket"; + private static final String KEY_NAME = "dir/file"; + private static final String DIR_NAME = "dir"; + private static final long OBJECT_ID = 10L; + private static final long PARENT_ID = 20L; + private static final long UPDATE_ID = 30L; + + @Test + void testKeyInfoParserSignatureAndUpdateId() throws Exception { + OmKeyInfo keyInfo = createKeyInfo(100L, 200L, 1024L, createChecksum((byte) 1), + createMetadata("meta", "one"), createTags("tag", "one"), createAcls(), + Collections.singletonList(createKeyLocationGroup(1L))); + byte[] rawData = OmKeyInfo.getCodec().toPersistedFormat(keyInfo); + + SnapshotDiffValueParser.ParsedRequiredInfo parsed = + SnapshotDiffValueParser.parseKeyInfoRequiredFields(rawData, true); + assertEquals(UPDATE_ID, parsed.getUpdateId()); + assertEquals(OBJECT_ID, parsed.getObjectId()); + assertEquals(PARENT_ID, parsed.getParentId()); + assertEquals(KEY_NAME, parsed.getName()); + assertFalse(SnapshotDiffValueParser.parseKeyInfoRequiredFields(rawData, false).hasUpdateId()); + + OmKeyInfo metadataChanged = createKeyInfo(100L, 200L, 1024L, createChecksum((byte) 1), + createMetadata("meta", "two"), createTags("tag", "one"), createAcls(), + Collections.singletonList(createKeyLocationGroup(1L))); + byte[] metadataRaw = OmKeyInfo.getCodec().toPersistedFormat(metadataChanged); + assertFalse(Arrays.equals(SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawData), + SnapshotDiffValueParser.computeKeyInfoCompareSignature(metadataRaw))); + + Map hsyncMetadata = createMetadata("meta", "one"); + hsyncMetadata.put(OzoneConsts.HSYNC_CLIENT_ID, "client1"); + OmKeyInfo hsyncChanged = createKeyInfo(100L, 200L, 1024L, createChecksum((byte) 1), + hsyncMetadata, createTags("tag", "one"), createAcls(), + Collections.singletonList(createKeyLocationGroup(1L))); + byte[] hsyncRaw = OmKeyInfo.getCodec().toPersistedFormat(hsyncChanged); + assertFalse(Arrays.equals(SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawData), + SnapshotDiffValueParser.computeKeyInfoCompareSignature(hsyncRaw))); + } + + @Test + void testKeyInfoIgnoresVolatileTimes() throws Exception { + OmKeyInfo keyInfo = createKeyInfo(100L, 200L, 1024L, createChecksum((byte) 1), + createMetadata("meta", "one"), createTags("tag", "one"), createAcls(), + Collections.singletonList(createKeyLocationGroup(1L))); + OmKeyInfo timeChanged = createKeyInfo(110L, 220L, 1024L, createChecksum((byte) 1), + createMetadata("meta", "one"), createTags("tag", "one"), createAcls(), + Collections.singletonList(createKeyLocationGroup(1L))); + + byte[] rawData = OmKeyInfo.getCodec().toPersistedFormat(keyInfo); + byte[] rawTimeChanged = OmKeyInfo.getCodec().toPersistedFormat(timeChanged); + + assertArrayEquals( + SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawData), + SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawTimeChanged)); + } + + @Test + void testDirectoryInfoSignatureAndParsing() throws Exception { + OmDirectoryInfo dirInfo = createDirectoryInfo(100L, 200L, createMetadata("meta", "one"), createAcls()); + byte[] rawData = OmDirectoryInfo.getCodec().toPersistedFormat(dirInfo); + SnapshotDiffValueParser.ParsedRequiredInfo parsed = + SnapshotDiffValueParser.parseDirectoryInfoRequiredFields(rawData, true); + assertEquals(UPDATE_ID, parsed.getUpdateId()); + assertEquals(OBJECT_ID, parsed.getObjectId()); + assertEquals(PARENT_ID, parsed.getParentId()); + assertEquals(DIR_NAME, parsed.getName()); + assertFalse(SnapshotDiffValueParser.parseDirectoryInfoRequiredFields(rawData, false).hasUpdateId()); + + OmDirectoryInfo metadataChanged = createDirectoryInfo(100L, 200L, createMetadata("meta", "two"), createAcls()); + byte[] rawMetadataChanged = OmDirectoryInfo.getCodec().toPersistedFormat(metadataChanged); + assertFalse(Arrays.equals(SnapshotDiffValueParser.computeDirectoryInfoCompareSignature(rawData), + SnapshotDiffValueParser.computeDirectoryInfoCompareSignature(rawMetadataChanged))); + + OmDirectoryInfo timeChanged = createDirectoryInfo(110L, 220L, createMetadata("meta", "one"), createAcls()); + byte[] rawTimeChanged = OmDirectoryInfo.getCodec().toPersistedFormat(timeChanged); + assertArrayEquals( + SnapshotDiffValueParser.computeDirectoryInfoCompareSignature(rawData), + SnapshotDiffValueParser.computeDirectoryInfoCompareSignature(rawTimeChanged)); + } + + @SuppressWarnings("checkstyle:ParameterNumber") + private static OmKeyInfo createKeyInfo(long creationTime, long modificationTime, long dataSize, FileChecksum checksum, + Map metadata, Map tags, List acls, + List keyLocationGroups) { + return new OmKeyInfo.Builder() + .setVolumeName(VOLUME) + .setBucketName(BUCKET) + .setKeyName(KEY_NAME) + .setCreationTime(creationTime) + .setModificationTime(modificationTime) + .setReplicationConfig(RatisReplicationConfig.getInstance(ReplicationFactor.ONE)) + .setObjectID(OBJECT_ID) + .setParentObjectID(PARENT_ID) + .setUpdateID(UPDATE_ID) + .setDataSize(dataSize) + .setOmKeyLocationInfos(keyLocationGroups) + .setFileChecksum(checksum) + .addAllMetadata(metadata) + .setTags(tags) + .setAcls(acls) + .build(); + } + + private static OmDirectoryInfo createDirectoryInfo(long creationTime, long modificationTime, + Map metadata, List acls) { + return OmDirectoryInfo.newBuilder() + .setName(DIR_NAME) + .setCreationTime(creationTime) + .setModificationTime(modificationTime) + .setObjectID(OBJECT_ID) + .setParentObjectID(PARENT_ID) + .setUpdateID(UPDATE_ID) + .addAllMetadata(metadata) + .setAcls(acls) + .build(); + } + + private static OmKeyLocationInfoGroup createKeyLocationGroup(long blockId) { + OmKeyLocationInfo location = new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(blockId, blockId)) + .build(); + return new OmKeyLocationInfoGroup(0, Collections.singletonList(location)); + } + + private static FileChecksum createChecksum(byte value) { + byte[] bytes = new byte[32]; + Arrays.fill(bytes, value); + MD5Hash fileMd5 = MD5Hash.digest(bytes); + return new MD5MD5CRC32GzipFileChecksum(0, 0, fileMd5); + } + + private static Map createMetadata(String key, String value) { + Map metadata = new LinkedHashMap<>(); + metadata.put(key, value); + return metadata; + } + + private static Map createTags(String key, String value) { + Map tags = new LinkedHashMap<>(); + tags.put(key, value); + return tags; + } + + private static List createAcls() { + return Collections.singletonList(OzoneAcl.parseAcl("user:test:rw")); + } +} From f263751f2389949c9e60a86959c140d2c89c38ed Mon Sep 17 00:00:00 2001 From: SaketaChalamchala Date: Thu, 25 Jun 2026 16:19:45 -0700 Subject: [PATCH 2/3] HDDS-15387. Addressed comments. --- .../om/snapshot/SnapshotDiffValueParser.java | 85 ++++++++++++------- .../snapshot/TestSnapshotDiffValueParser.java | 58 +++++++++++++ 2 files changed, 110 insertions(+), 33 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffValueParser.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffValueParser.java index 58d193ba17c6..90082c7b64f2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffValueParser.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffValueParser.java @@ -17,14 +17,15 @@ package org.apache.hadoop.ozone.om.snapshot; -import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitKeyRequest.HSYNC_FIELD_NUMBER; - import com.google.protobuf.ByteString; import com.google.protobuf.CodedInputStream; import com.google.protobuf.WireFormat; import java.io.IOException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.KeyValue; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DirectoryInfo; @@ -36,6 +37,7 @@ public final class SnapshotDiffValueParser { private static final int INT_BYTES = 4; private static final int LONG_BYTES = 8; + private static final int HSYNC_METADATA_PRESENT_TAG = 1001; private static final String DIGEST_ALGORITHM = "SHA-256"; private SnapshotDiffValueParser() { @@ -83,9 +85,11 @@ public static ParsedRequiredInfo parseKeyInfoRequiredFields(byte[] value, boolea public static byte[] computeKeyInfoCompareSignature(byte[] value) throws IOException { CodedInputStream input = CodedInputStream.newInstance(value); MessageDigest digest = newDigest(); - boolean hasHsyncMetadata = false; + AtomicBoolean hasHsyncMetadata = new AtomicBoolean(false); int keyLocationListCount = 0; ByteString latestKeyLocationList = null; + List metadataSignatures = new ArrayList<>(); + List tagSignatures = new ArrayList<>(); int tag; while ((tag = input.readTag()) != 0) { @@ -99,18 +103,18 @@ public static byte[] computeKeyInfoCompareSignature(byte[] value) throws IOExcep keyLocationListCount++; break; case KeyInfo.METADATA_FIELD_NUMBER: - MetadataEntry metadataEntry = parseMetadataEntry(input.readBytes().toByteArray()); - if (metadataEntry != null) { - updateDigestWithRawBytes(digest, fieldNumber, metadataEntry.getDigest()); - if (metadataEntry.hasHsync()) { - hasHsyncMetadata = true; - } + byte[] metadataDigest = parseKeyValueDigest(input.readBytes().toByteArray(), true, hasHsyncMetadata); + if (metadataDigest != null) { + metadataSignatures.add(metadataDigest); } break; case KeyInfo.FILECHECKSUM_FIELD_NUMBER: case KeyInfo.ACLS_FIELD_NUMBER: case KeyInfo.TAGS_FIELD_NUMBER: - updateDigestWithBytes(digest, fieldNumber, input.readBytes()); + byte[] tagDigest = parseKeyValueDigest(input.readBytes().toByteArray(), false, null); + if (tagDigest != null) { + tagSignatures.add(tagDigest); + } break; default: input.skipField(tag); @@ -121,7 +125,9 @@ public static byte[] computeKeyInfoCompareSignature(byte[] value) throws IOExcep if (latestKeyLocationList != null) { updateDigestWithBytes(digest, KeyInfo.KEYLOCATIONLIST_FIELD_NUMBER, latestKeyLocationList); } - updateDigestWithBoolean(digest, HSYNC_FIELD_NUMBER, hasHsyncMetadata); + addCanonicalizedDigest(digest, KeyInfo.METADATA_FIELD_NUMBER, metadataSignatures); + addCanonicalizedDigest(digest, KeyInfo.TAGS_FIELD_NUMBER, tagSignatures); + updateDigestWithBoolean(digest, HSYNC_METADATA_PRESENT_TAG, hasHsyncMetadata.get()); updateDigestWithInt(digest, keyLocationListCount); return digest.digest(); @@ -169,15 +175,16 @@ public static ParsedRequiredInfo parseDirectoryInfoRequiredFields(byte[] value, public static byte[] computeDirectoryInfoCompareSignature(byte[] value) throws IOException { CodedInputStream input = CodedInputStream.newInstance(value); MessageDigest digest = newDigest(); + List metadataSignatures = new ArrayList<>(); int tag; while ((tag = input.readTag()) != 0) { int fieldNumber = WireFormat.getTagFieldNumber(tag); switch (fieldNumber) { case DirectoryInfo.METADATA_FIELD_NUMBER: - MetadataEntry metadataEntry = parseMetadataEntry(input.readBytes().toByteArray()); - if (metadataEntry != null) { - updateDigestWithRawBytes(digest, fieldNumber, metadataEntry.getDigest()); + byte[] metadataDigest = parseKeyValueDigest(input.readBytes().toByteArray(), false, null); + if (metadataDigest != null) { + metadataSignatures.add(metadataDigest); } break; case DirectoryInfo.ACLS_FIELD_NUMBER: @@ -189,6 +196,8 @@ public static byte[] computeDirectoryInfoCompareSignature(byte[] value) throws I } } + addCanonicalizedDigest(digest, KeyInfo.METADATA_FIELD_NUMBER, metadataSignatures); + return digest.digest(); } @@ -247,11 +256,25 @@ private static MessageDigest newDigest() { } } - private static MetadataEntry parseMetadataEntry(byte[] metadataBytes) throws IOException { - if (metadataBytes == null || metadataBytes.length == 0) { + private static void addCanonicalizedDigest( + MessageDigest digest, int fieldNumber, List metadataEntries) { + if (metadataEntries.isEmpty()) { + return; + } + metadataEntries.sort(SnapshotDiffValueParser::compareBytes); + MessageDigest metadataDigest = newDigest(); + for (byte[] metadataEntry : metadataEntries) { + metadataDigest.update(metadataEntry); + } + updateDigestWithRawBytes(digest, fieldNumber, metadataDigest.digest()); + } + + private static byte[] parseKeyValueDigest(byte[] keyValueBytes, boolean computeHsync, AtomicBoolean hasHsync) + throws IOException { + if (keyValueBytes == null || keyValueBytes.length == 0) { return null; } - CodedInputStream input = CodedInputStream.newInstance(metadataBytes); + CodedInputStream input = CodedInputStream.newInstance(keyValueBytes); String key = null; String value = null; while (!input.isAtEnd()) { @@ -272,28 +295,24 @@ private static MetadataEntry parseMetadataEntry(byte[] metadataBytes) throws IOE break; } } + if (computeHsync && hasHsync != null && OzoneConsts.HSYNC_CLIENT_ID.equals(key)) { + hasHsync.set(true); + } MessageDigest entryDigest = newDigest(); updateTaggedString(entryDigest, KeyValue.KEY_FIELD_NUMBER, key); updateTaggedString(entryDigest, KeyValue.VALUE_FIELD_NUMBER, value); - return new MetadataEntry(entryDigest.digest(), OzoneConsts.HSYNC_CLIENT_ID.equals(key)); + return entryDigest.digest(); } - private static final class MetadataEntry { - private final byte[] digest; - private final boolean hasHsync; - - private MetadataEntry(byte[] digest, boolean hasHsync) { - this.digest = digest; - this.hasHsync = hasHsync; - } - - private byte[] getDigest() { - return digest; - } - - private boolean hasHsync() { - return hasHsync; + private static int compareBytes(byte[] left, byte[] right) { + int length = Math.min(left.length, right.length); + for (int i = 0; i < length; i++) { + int diff = (left[i] & 0xFF) - (right[i] & 0xFF); + if (diff != 0) { + return diff; + } } + return left.length - right.length; } /** diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffValueParser.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffValueParser.java index 292227cb1b55..261a97a130d8 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffValueParser.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffValueParser.java @@ -79,6 +79,48 @@ hsyncMetadata, createTags("tag", "one"), createAcls(), byte[] hsyncRaw = OmKeyInfo.getCodec().toPersistedFormat(hsyncChanged); assertFalse(Arrays.equals(SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawData), SnapshotDiffValueParser.computeKeyInfoCompareSignature(hsyncRaw))); + + OmKeyInfo tagsChanged = createKeyInfo(100L, 200L, 1024L, createChecksum((byte) 1), + createMetadata("meta", "one"), createTags("tag", "two"), createAcls(), + Collections.singletonList(createKeyLocationGroup(1L))); + byte[] tagsRaw = OmKeyInfo.getCodec().toPersistedFormat(tagsChanged); + assertFalse(Arrays.equals(SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawData), + SnapshotDiffValueParser.computeKeyInfoCompareSignature(tagsRaw))); + + OmKeyInfo aclsChanged = createKeyInfo(100L, 200L, 1024L, createChecksum((byte) 1), + createMetadata("meta", "one"), createTags("tag", "one"), createAcls("user:other:rw"), + Collections.singletonList(createKeyLocationGroup(1L))); + byte[] aclsRaw = OmKeyInfo.getCodec().toPersistedFormat(aclsChanged); + assertFalse(Arrays.equals(SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawData), + SnapshotDiffValueParser.computeKeyInfoCompareSignature(aclsRaw))); + + OmKeyInfo checksumChanged = createKeyInfo(100L, 200L, 1024L, createChecksum((byte) 2), + createMetadata("meta", "one"), createTags("tag", "one"), createAcls(), + Collections.singletonList(createKeyLocationGroup(1L))); + byte[] checksumRaw = OmKeyInfo.getCodec().toPersistedFormat(checksumChanged); + assertFalse(Arrays.equals(SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawData), + SnapshotDiffValueParser.computeKeyInfoCompareSignature(checksumRaw))); + + OmKeyInfo dataSizeChanged = createKeyInfo(100L, 200L, 2048L, createChecksum((byte) 1), + createMetadata("meta", "one"), createTags("tag", "one"), createAcls(), + Collections.singletonList(createKeyLocationGroup(1L))); + byte[] dataSizeRaw = OmKeyInfo.getCodec().toPersistedFormat(dataSizeChanged); + assertFalse(Arrays.equals(SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawData), + SnapshotDiffValueParser.computeKeyInfoCompareSignature(dataSizeRaw))); + + OmKeyInfo locationCountChanged = createKeyInfo(100L, 200L, 1024L, createChecksum((byte) 1), + createMetadata("meta", "one"), createTags("tag", "one"), createAcls(), + createKeyLocationGroups(1L, 2L)); + byte[] locationCountRaw = OmKeyInfo.getCodec().toPersistedFormat(locationCountChanged); + assertFalse(Arrays.equals(SnapshotDiffValueParser.computeKeyInfoCompareSignature(rawData), + SnapshotDiffValueParser.computeKeyInfoCompareSignature(locationCountRaw))); + + OmKeyInfo latestLocationChanged = createKeyInfo(100L, 200L, 1024L, createChecksum((byte) 1), + createMetadata("meta", "one"), createTags("tag", "one"), createAcls(), + createKeyLocationGroups(1L, 3L)); + byte[] latestLocationRaw = OmKeyInfo.getCodec().toPersistedFormat(latestLocationChanged); + assertFalse(Arrays.equals(SnapshotDiffValueParser.computeKeyInfoCompareSignature(locationCountRaw), + SnapshotDiffValueParser.computeKeyInfoCompareSignature(latestLocationRaw))); } @Test @@ -166,6 +208,18 @@ private static OmKeyLocationInfoGroup createKeyLocationGroup(long blockId) { return new OmKeyLocationInfoGroup(0, Collections.singletonList(location)); } + private static List createKeyLocationGroups(long... blockIds) { + List groups = new java.util.ArrayList<>(); + int version = 0; + for (long blockId : blockIds) { + OmKeyLocationInfo location = new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(blockId, blockId)) + .build(); + groups.add(new OmKeyLocationInfoGroup(version++, Collections.singletonList(location))); + } + return groups; + } + private static FileChecksum createChecksum(byte value) { byte[] bytes = new byte[32]; Arrays.fill(bytes, value); @@ -188,4 +242,8 @@ private static Map createTags(String key, String value) { private static List createAcls() { return Collections.singletonList(OzoneAcl.parseAcl("user:test:rw")); } + + private static List createAcls(String acl) { + return Collections.singletonList(OzoneAcl.parseAcl(acl)); + } } From 4758f9b31a38f8fdd0943b7ce61afc4bf9445df5 Mon Sep 17 00:00:00 2001 From: SaketaChalamchala Date: Fri, 26 Jun 2026 14:38:18 -0700 Subject: [PATCH 3/3] HDDS-15387. Fixed unit test. --- .../hadoop/ozone/om/snapshot/SnapshotDiffValueParser.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffValueParser.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffValueParser.java index 90082c7b64f2..48885a2942f1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffValueParser.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffValueParser.java @@ -110,6 +110,8 @@ public static byte[] computeKeyInfoCompareSignature(byte[] value) throws IOExcep break; case KeyInfo.FILECHECKSUM_FIELD_NUMBER: case KeyInfo.ACLS_FIELD_NUMBER: + updateDigestWithBytes(digest, fieldNumber, input.readBytes()); + break; case KeyInfo.TAGS_FIELD_NUMBER: byte[] tagDigest = parseKeyValueDigest(input.readBytes().toByteArray(), false, null); if (tagDigest != null) {