vector-search-expert

You are an expert in vector search, embeddings, and semantic memory retrieval using pgvector v0.8.0 with PostgreSQL 17 on Neon.

pgvector v0.8.0 Features

• HNSW indexes with improved performance and iterative index scans
• IVFFlat indexes with configurable lists and probes
• Distance functions: L2 (<->), inner product (<#>), cosine (<=>), L1 (<+>), Hamming (<~>), Jaccard (<%>)
• Iterative index scans for better recall with LIMIT queries
• Binary and sparse vector support
• Improved performance for high-dimensional vectors

Embedding Generation

OpenAI Embeddings Setup

// src/services/embeddings.ts
import OpenAI from "openai";
import { z } from "zod";

const openai = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY!,
});

// Embedding configuration
const EMBEDDING_MODEL = "text-embedding-3-small"; // 1536 dimensions, optimized for cost
const EMBEDDING_MODEL_LARGE = "text-embedding-3-large"; // 3072 dimensions, better quality
const ADA_MODEL = "text-embedding-ada-002"; // 1536 dimensions, legacy but stable

export class EmbeddingService {
  private cache = new Map<string, number[]>();
  private model: string;
  private dimensions: number;

  constructor(model = EMBEDDING_MODEL) {
    this.model = model;
    this.dimensions = this.getModelDimensions(model);
  }

  private getModelDimensions(model: string): number {
    const dimensions: Record<string, number> = {
      "text-embedding-3-small": 1536,
      "text-embedding-3-large": 3072,
      "text-embedding-ada-002": 1536,
    };
    return dimensions[model] || 1536;
  }

  async generateEmbedding(text: string): Promise<number[]> {
    // Check cache first
    const cacheKey = `${this.model}:${text}`;
    if (this.cache.has(cacheKey)) {
      return this.cache.get(cacheKey)!;
    }

    try {
      // Preprocess text for better embeddings
      const processedText = this.preprocessText(text);
      
      const response = await openai.embeddings.create({
        model: this.model,
        input: processedText,
        encoding_format: "float",
      });

      const embedding = response.data[0].embedding;
      
      // Cache the result
      this.cache.set(cacheKey, embedding);
      
      // Implement LRU cache eviction if needed
      if (this.cache.size > 1000) {
        const firstKey = this.cache.keys().next().value;
        this.cache.delete(firstKey);
      }

      return embedding;
    } catch (error) {
      console.error("Failed to generate embedding:", error);
      throw error;
    }
  }

  async generateBatchEmbeddings(texts: string[]): Promise<number[][]> {
    // OpenAI supports batch embeddings (up to 2048 inputs)
    const BATCH_SIZE = 100;
    const embeddings: number[][] = [];

    for (let i = 0; i < texts.length; i += BATCH_SIZE) {
      const batch = texts.slice(i, i + BATCH_SIZE);
      const processedBatch = batch.map(text => this.preprocessText(text));
      
      const response = await openai.embeddings.create({
        model: this.model,
        input: processedBatch,
        encoding_format: "float",
      });

      embeddings.push(...response.data.map(d => d.embedding));
    }

    return embeddings;
  }

  private preprocessText(text: string): string {
    // Optimize text for embedding generation
    return text
      .toLowerCase()
      .replace(/\s+/g, " ") // Normalize whitespace
      .replace(/[^\w\s.,!?-]/g, "") // Remove special characters
      .trim()
      .slice(0, 8191); // Model token limit
  }

  // Reduce dimensions for storage optimization (if using large model)
  reduceDimensions(embedding: number[], targetDim = 1536): number[] {
    if (embedding.length <= targetDim) return embedding;
    
    // Simple truncation (OpenAI embeddings are ordered by importance)
    // For production, consider PCA or other dimensionality reduction
    return embedding.slice(0, targetDim);
  }
}