pip install vecta[all]

from vecta import VectaAPIClient

client = VectaAPIClient(api_key="your-api-key")

# Upload files directly (simplest approach)
data_source = client.upload_local_files([
    "docs/manual.pdf",
    "docs/guide.docx",
    "docs/faq.txt"
])

# Generate a synthetic Q&A dataset from your uploaded files
benchmark = client.create_benchmark(
    data_source_id=data_source["id"],
    questions_count=10,
)

# This is your RAG pipeline
def my_rag(query: str) -> tuple[list[str], str]:
    # ...
    return retrieved_chunk_ids, generated_answer

# Run evaluation
results = client.evaluate_retrieval_and_generation(
    benchmark_id=benchmark["id"],
    retrieval_generation_function=my_rag,
    evaluation_name="my-first-eval"
)

print(f"F1 Score: {results.document_level.f1_score:.3f}")

from vecta import VectaAPIClient
from vecta.core.schemas import VectorDBSchema

client = VectaAPIClient(api_key="your-api-key")

# Define how to read results from your vector database
schema = VectorDBSchema(
    id_accessor="id",
    content_accessor="text",
    source_path_accessor="metadata.filename",
    page_nums_accessor="metadata.pages"
)

# ChromaDB Cloudmple
db = client.connect_chroma_cloud(
    tenant="your-tenant",
    database="your-db",
    api_key="your-chroma-key",
    collection_name="documents",
    schema=schema  # Optional: uses default if not provided
)

# Or Pinecone
db = client.connect_pinecone(
    api_key="your-pinecone-key",
    index_name="your-index",
    namespace="",  # Optional
    schema=schema   # Optional: uses default if not provided
)

# 10+ other vector databases supported, or
# easily make a custom connector if needed

# Then create benchmark from existing data
benchmark = client.create_benchmark(
    data_source_id=db["id"],
    questions_count=50,
)

pip install vecta[all]

from vecta import VectaClient
from vecta.connectors.file_store_connector import FileStoreConnector

# Process local files (simplest approach)
connector = FileStoreConnector(
    file_paths=[
        "manual.pdf",
        "guide.docx"
    ],
    base_path="/path/to/files"
)

# Initialize Vecta (or set OPENROUTER_API_KEY environment variable)
vecta = VectaClient(
    data_source_connector=connector,
    openai_api_key="your-openai-key",
    openai_base_url="https://api.openai.com/v1",  # or your custom LLM server
    model="gpt-4o"
)

# Load your files
vecta.load_knowledge_base()

# Generate benchmark
entries = vecta.generate_benchmark(
    n_questions=50,
    random_seed=42
)

# Save for later
vecta.save_benchmark("benchmark.csv")

# Test your retriever
def my_retriever(query: str) -> list[str]:
    results = connector.semantic_search(query, k=10)
    return [r.id for r in results]

results = vecta.evaluate_retrieval(my_retriever)
print(f"F1: {results.chunk_level.f1_score:.3f}")

from vecta import VectaClient
from vecta.connectors.chroma_local_connector import ChromaLocalConnector
from vecta.core.schema_helpers import SchemaTemplates
from vecta.core.schemas import VectorDBSchema
from chromadb import Client

# Connect to existing ChromaDB
chroma = Client()

# Use schema template or define custom schema
schema = SchemaTemplates.chroma_default()
# Or create custom schema:
# schema = VectorDBSchema(
#     id_accessor="id",
#     content_accessor="document", 
#     source_path_accessor="metadata.source_path",
#     page_nums_accessor="metadata.page_nums"
# )

connector = ChromaLocalConnector(
    client=chroma,
    collection_name="documents",
    schema=schema  # Schema parameter is required
)

# Initialize Vecta (or set OPENROUTER_API_KEY environment variable)
vecta = VectaClient(
    data_source_connector=connector,
    openai_api_key="your-openai-key",
    openai_base_url="https://api.openai.com/v1",  # or your custom LLM server
    model="gpt-4o"
)

def my_retriever(query: str) -> list[str]:
    results = vector_db.search(query, k=10)
    return [r.id for r in results]

results = client.evaluate_retrieval(
    benchmark_id=benchmark["id"],
    retrieval_function=my_retriever
)

def my_rag(query: str) -> tuple[list[str], str]:
    # Retrieve
    chunks = vector_db.search(query, k=5)
    chunk_ids = [c.id for c in chunks]
    
    # Generate
    context = "\n".join([c.content for c in chunks])
    answer = llm.generate(f"Context: {context}\n\nQ: {query}")
    
    return chunk_ids, answer

# Evaluate
results = client.evaluate_retrieval_and_generation(
    benchmark_id=benchmark["id"],
    retrieval_generation_function=my_rag
)

def my_generator(query: str) -> str:
    return llm.generate(query)

results = client.evaluate_generation_only(
    benchmark_id=benchmark["id"],
    generation_function=my_generator
)

from vecta.core.dataset_importer import BenchmarkDatasetImporter

importer = BenchmarkDatasetImporter()
chunks, entries = importer.import_gpqa_diamond(max_items=50)

vecta.benchmark_entries = entries
results = vecta.evaluate_generation_only(my_generator)