Quickstart

def my_rag(query: str) -> tuple[list[str], str]:
    # ...
    return retrieved_chunk_ids, answer

from vecta import VectaAPIClient

client = VectaAPIClient() 

data_source = client.upload_local_files(
    file_paths=["knowledge_base.pdf", "faq.docx"],
) # you can also connect your vector db for more granular results

benchmark = client.create_benchmark(
    data_source_id=data_source["id"],
    questions_count=10,
) # you can also load custom benchmarks, including from huggingface

results = client.evaluate_retrieval_and_generation(
    benchmark_id=benchmark["id"],
    retrieval_generation_function=my_rag,
)

print(f"Retriever F1: {results.document_level.f1_score}")
print(f"Response Accuracy: {results.generation_metrics.accuracy}")
print(f"Groundedness: {results.generation_metrics.groundedness}")