glove-100 example added.

examples/glove-100/README

@@ -0,0 +1,3 @@
+wget http://ann-benchmarks.com/glove-100-angular.hdf5
+python insert.py
+python recall.py (use --k <count> optionally, default top-10)

examples/glove-100/insert.py

@@ -0,0 +1,47 @@
+import h5py
+import redis
+from tqdm import tqdm
+
+# Initialize Redis connection
+redis_client = redis.Redis(host='localhost', port=6379, decode_responses=True, encoding='utf-8')
+
+def add_to_redis(index, embedding):
+    """Add embedding to Redis using VADD command"""
+    args = ["VADD", "glove_embeddings", "VALUES", "100"]  # 100 is vector dimension
+    args.extend(map(str, embedding))
+    args.append(f"{index}")  # Using index as identifier since we don't have words
+    args.append("EF")
+    args.append("200")
+    # args.append("NOQUANT")
+    # args.append("BIN")
+    redis_client.execute_command(*args)
+
+def main():
+    with h5py.File('glove-100-angular.hdf5', 'r') as f:
+        # Get the train dataset
+        train_vectors = f['train']
+        total_vectors = train_vectors.shape[0]
+
+        print(f"Starting to process {total_vectors} vectors...")
+
+        # Process in batches to avoid memory issues
+        batch_size = 1000
+
+        for i in tqdm(range(0, total_vectors, batch_size)):
+            batch_end = min(i + batch_size, total_vectors)
+            batch = train_vectors[i:batch_end]
+
+            for j, vector in enumerate(batch):
+                try:
+                    current_index = i + j
+                    add_to_redis(current_index, vector)
+
+                except Exception as e:
+                    print(f"Error processing vector {current_index}: {str(e)}")
+                    continue
+
+            if (i + batch_size) % 10000 == 0:
+                print(f"Processed {i + batch_size} vectors")
+
+if __name__ == "__main__":
+    main()

examples/glove-100/recall.py

@@ -0,0 +1,78 @@
+import h5py
+import redis
+import numpy as np
+from tqdm import tqdm
+import argparse
+
+# Initialize Redis connection
+redis_client = redis.Redis(host='localhost', port=6379, decode_responses=True, encoding='utf-8')
+
+def get_redis_neighbors(query_vector, k):
+    """Get nearest neighbors using Redis VSIM command"""
+    args = ["VSIM", "glove_embeddings_bin", "VALUES", "100"]
+    args.extend(map(str, query_vector))
+    args.extend(["COUNT", str(k)])
+    args.extend(["EF", 100])
+    if False:
+        print(args)
+        exit(1)
+    results = redis_client.execute_command(*args)
+    return [int(res) for res in results]
+
+def calculate_recall(ground_truth, predicted, k):
+    """Calculate recall@k"""
+    relevant = set(ground_truth[:k])
+    retrieved = set(predicted[:k])
+    return len(relevant.intersection(retrieved)) / len(relevant)
+
+def main():
+    parser = argparse.ArgumentParser(description='Evaluate Redis VSIM recall')
+    parser.add_argument('--k', type=int, default=10, help='Number of neighbors to evaluate (default: 10)')
+    parser.add_argument('--batch', type=int, default=100, help='Progress update frequency (default: 100)')
+    args = parser.parse_args()
+
+    k = args.k
+    batch_size = args.batch
+
+    with h5py.File('glove-100-angular.hdf5', 'r') as f:
+        test_vectors = f['test'][:]
+        ground_truth_neighbors = f['neighbors'][:]
+        
+        num_queries = len(test_vectors)
+        recalls = []
+        
+        print(f"Evaluating recall@{k} for {num_queries} test queries...")
+        
+        for i in tqdm(range(num_queries)):
+            try:
+                # Get Redis results
+                redis_neighbors = get_redis_neighbors(test_vectors[i], k)
+                
+                # Get ground truth for this query
+                true_neighbors = ground_truth_neighbors[i]
+                
+                # Calculate recall
+                recall = calculate_recall(true_neighbors, redis_neighbors, k)
+                recalls.append(recall)
+                
+                if (i + 1) % batch_size == 0:
+                    current_avg_recall = np.mean(recalls)
+                    print(f"Current average recall@{k} after {i+1} queries: {current_avg_recall:.4f}")
+                
+            except Exception as e:
+                print(f"Error processing query {i}: {str(e)}")
+                continue
+        
+        final_recall = np.mean(recalls)
+        print("\nFinal Results:")
+        print(f"Average recall@{k}: {final_recall:.4f}")
+        print(f"Total queries evaluated: {len(recalls)}")
+        
+        # Save detailed results
+        with open(f'recall_evaluation_results_k{k}.txt', 'w') as f:
+            f.write(f"Average recall@{k}: {final_recall:.4f}\n")
+            f.write(f"Total queries evaluated: {len(recalls)}\n")
+            f.write(f"Individual query recalls: {recalls}\n")
+
+if __name__ == "__main__":
+    main()