Use xxh64 by default and remove the 2 bytes prefix from the hashes.bin for even better compression

Author: Toflar

README.md

@@ -35,11 +35,11 @@ Typical use cases:
 FastSet uses a **sorted fingerprint array** instead of storing the original strings.
 
 ### Build time
-1. Read a newline-separated dictionary file.
-2. Hash each entry using **`xxh128`** (16 bytes, raw).
+1. Read to original dictionary built using the `SetBuilder`
+2. Hash each entry using the configured hash
 3. Sort all fingerprints.
 4. Write two files:
-   - `hashes.bin` – concatenated 16-byte fingerprints
+   - `hashes.bin` – concatenated fingerprints (without the 2-byte prefixes for maximum compression)
    - `index.bin` – a 2-byte prefix index (65,537 offsets)
 
 ### Lookup time
@@ -48,15 +48,16 @@ FastSet uses a **sorted fingerprint array** instead of storing the original stri
 3. Binary-search **only inside that bucket**.
 
 Because hash prefixes are uniformly distributed, buckets are tiny.
-Real world tests on the fixture file that contains ~140k entries in this repository (`tests/Fixtures/terms_de.txt`):
+Real world tests on the fixture file that contains ~140k entries in this repository (`tests/Fixtures/terms_de.txt`)
+using the `xxh128` hash algorithm:
 
 - Average bucket size `≈2–3` entries
 - Worst case (real data): `11` entries
 - Worst case comparisons: `log₂(11) = 4`
 
 Of course, the bigger your dictionary, the bigger the individual buckets.
 
-All comparisons are fixed-width (16 bytes), not variable-length UTF-8 strings.
+All comparisons are fixed-width, not variable-length UTF-8 strings.
 
 ---
 
@@ -117,9 +118,13 @@ SetBuilder::buildSet($myOriginalSet, './compressed.gz');
 // array directly:
 SetBuilder::buildFromArray($myArray, './compressed.gz');
 
+// You can use either xxh64 (default) or xxh128
+// xxh128 will have a smaller probability for false-positives but require more memory
+$hashAlgorithm = 'xxh64';
+
 // You then ship either "compressed.txt" or "compressed.gz" with your application. Instantiating 
 // is then done as follows:
-$set = new FastSet(__DIR__ . '/dict');
+$set = new FastSet(__DIR__ . '/dict', $hashAlgorithm);
 $set->build(__DIR__ . '/compressed.(txt|gz)'); // Must be a file built using the SetBuilder
 ```
 

src/FastSet.php

@@ -10,6 +10,10 @@ final class FastSet
 
     private readonly string $indexPath;
 
+    private readonly int $fingerprintByteLength;
+
+    private readonly int $storedTailByteLength;
+
     private bool $isInitialized = false;
 
     private string $blob = '';
@@ -33,18 +37,27 @@ final class FastSet
      */
     private array $prefixOffsets = [];
 
-    public function __construct(private readonly string $directory)
-    {
-        $this->hashesPath = $this->directory.'/hashes.bin';
-        $this->indexPath = $this->directory.'/index.bin';
-
+    public function __construct(
+        private readonly string $directory,
+        private readonly string $hashAlgorithm = 'xxh64',
+    ) {
         if (!is_dir($this->directory)) {
             throw new \InvalidArgumentException('Directory does not exist.');
         }
 
-        if (!\in_array('xxh128', hash_algos(), true)) {
-            throw new \LogicException('Requires xxh128 hash algorithm.');
+        if (!\in_array($this->hashAlgorithm, ['xxh64', 'xxh128'], true)) {
+            throw new \LogicException(\sprintf('Unsupported hash algorithm "%s". Use "xxh64" or "xxh128".', $this->hashAlgorithm));
+        }
+
+        if (!\in_array($this->hashAlgorithm, hash_algos(), true)) {
+            throw new \LogicException('Desired hash algorithm is not available.');
         }
+
+        $this->fingerprintByteLength = 'xxh128' === $this->hashAlgorithm ? 16 : 8;
+        $this->storedTailByteLength = $this->fingerprintByteLength - 2;
+
+        $this->indexPath = $this->directory.'/index_'.$this->hashAlgorithm.'.bin';
+        $this->hashesPath = $this->directory.'/hashes_'.$this->hashAlgorithm.'.bin';
     }
 
     public function has(string $entry): bool
@@ -68,15 +81,19 @@ public function has(string $entry): bool
             return false;
         }
 
+        $queryFingerprintTailBytes = substr($fingerprint, 2, $this->storedTailByteLength);
+
         // Binary search within that bucket
         $low = $startIndex;
         $high = $endIndex - 1;
 
         while ($low <= $high) {
             $mid = $low + $high >> 1;
-            $midFingerprint = substr($this->blob, $mid * 16, 16);
 
-            $cmp = strcmp($midFingerprint, $fingerprint);
+            $middleTailByteOffset = $mid * $this->storedTailByteLength;
+            $middleFingerprintTailBytes = substr($this->blob, $middleTailByteOffset, $this->storedTailByteLength);
+
+            $cmp = strcmp($middleFingerprintTailBytes, $queryFingerprintTailBytes);
             if (0 === $cmp) {
                 return true;
             }
@@ -118,8 +135,9 @@ function (string $entry) use (&$fingerPrints): void {
             $prefixKey = $this->getPrefixKey($fingerprint);
             ++$prefixCounts[$prefixKey];
 
-            // Each fingerprint is exactly 16 bytes
-            fwrite($hashFile, $fingerprint);
+            // Skip the first 2 bytes in our hashes.bin - they are already part of the index.bin
+            // so we can save 2 bytes per fingerprint to reduce our memory footprint even more
+            fwrite($hashFile, substr($fingerprint, 2, $this->storedTailByteLength));
         }
 
         fclose($hashFile);
@@ -168,7 +186,7 @@ public function getPrefixKey(string $fingerprint): int
 
     private function getFingerPrintForEntry(string $entry): string
     {
-        return hash('xxh128', $entry, true);
+        return hash($this->hashAlgorithm, $entry, true);
     }
 
     /**

tests/FastSetTest.php

@@ -4,6 +4,7 @@
 
 namespace Toflar\FastSet\Tests;
 
+use PHPUnit\Framework\Attributes\TestWith;
 use PHPUnit\Framework\TestCase;
 use Symfony\Component\Filesystem\Filesystem;
 use Toflar\FastSet\FastSet;
@@ -31,21 +32,25 @@ public function testFastSetFailsWithWrongFileFormat(): void
         $fastSet->build(__DIR__.'/Fixtures/terms_de.txt');
     }
 
-    public function testWorkingWithSetBuilderWithoutGzipCompression(): void
+    #[TestWith(['xxh64'])]
+    #[TestWith(['xxh128'])]
+    public function testWorkingWithSetBuilderWithoutGzipCompression(string $hashAlgorithm): void
     {
         // Build a set without gzip but with our prefix algorithm
         SetBuilder::buildSet(__DIR__.'/Fixtures/terms_de.txt', $this->testDirectory.'/terms_encoded.txt');
 
         // File size of the encoded file must be definitely smaller
         $this->assertTrue(filesize($this->testDirectory.'/terms_encoded.txt') < filesize(__DIR__.'/Fixtures/terms_de.txt'));
 
-        $fastSet = new FastSet($this->testDirectory);
+        $fastSet = new FastSet($this->testDirectory, $hashAlgorithm);
         $fastSet->build($this->testDirectory.'/terms_encoded.txt');
 
         $this->assertFastSetContents($fastSet);
     }
 
-    public function testWorkingWithSetBuilderWithGzipCompression(): void
+    #[TestWith(['xxh64'])]
+    #[TestWith(['xxh128'])]
+    public function testWorkingWithSetBuilderWithGzipCompression(string $hashAlgorithm): void
     {
         // Build a set without gzip but with our prefix algorithm
         SetBuilder::buildSet(__DIR__.'/Fixtures/terms_de.txt', $this->testDirectory.'/terms_encoded.txt');
@@ -57,7 +62,7 @@ public function testWorkingWithSetBuilderWithGzipCompression(): void
         // File size of the gzipped file must be even smaller
         $this->assertTrue(filesize($this->testDirectory.'/terms_gzipped.gz') < filesize($this->testDirectory.'/terms_encoded.txt'));
 
-        $fastSet = new FastSet($this->testDirectory);
+        $fastSet = new FastSet($this->testDirectory, $hashAlgorithm);
         $fastSet->build($this->testDirectory.'/terms_gzipped.gz');
 
         $this->assertFastSetContents($fastSet);