vector-db-setup

Vector Database Setup

Configure vector databases for semantic search and AI applications.

Core Workflow

1. Choose database: Select based on requirements
2. Setup connection: Configure client
3. Generate embeddings: Create vector representations
4. Index documents: Store with metadata
5. Query vectors: Semantic similarity search
6. Optimize: Tune for performance

Database Comparison

Database	Type	Best For	Scaling
Pinecone	Managed	Production, no ops	Automatic
Chroma	Embedded/Server	Development, local	Manual
pgvector	PostgreSQL ext	Existing Postgres	With Postgres
Qdrant	Self-hosted	Full control	Manual
Weaviate	Managed/Self	GraphQL-like API	Both

Embeddings Generation

OpenAI Embeddings

// embeddings/openai.ts
import OpenAI from 'openai';

const openai = new OpenAI();

export async function generateEmbedding(text: string): Promise<number[]> {
  const response = await openai.embeddings.create({
    model: 'text-embedding-3-small', // or text-embedding-3-large
    input: text,
  });

  return response.data[0].embedding;
}

export async function generateEmbeddings(texts: string[]): Promise<number[][]> {
  const response = await openai.embeddings.create({
    model: 'text-embedding-3-small',
    input: texts,
  });

  return response.data.map((d) => d.embedding);
}

Batch Processing

// embeddings/batch.ts
const BATCH_SIZE = 100;

export async function batchGenerateEmbeddings(
  texts: string[]
): Promise<number[][]> {
  const embeddings: number[][] = [];

  for (let i = 0; i < texts.length; i += BATCH_SIZE) {
    const batch = texts.slice(i, i + BATCH_SIZE);
    const batchEmbeddings = await generateEmbeddings(batch);
    embeddings.push(...batchEmbeddings);

    // Rate limiting
    if (i + BATCH_SIZE < texts.length) {
      await new Promise((resolve) => setTimeout(resolve, 100));
    }
  }

  return embeddings;
}