Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions embeddings-and-vector-databases-with-chromadb/README.md
Original file line number Diff line number Diff line change
@@ -1,11 +1,18 @@
# Embeddings and Vector Databases With ChromaDB
# ChromaDB: Embeddings and Vector Databases in Python

Supporting code for the Real Python tutorial [Embeddings and Vector Databases With ChromaDB](https://realpython.com/chromadb-vector-database/).
Supporting code for the Real Python tutorial [ChromaDB: Embeddings and Vector Databases in Python](https://realpython.com/chromadb-vector-database/).

To run the code in this tutorial, you should have `numpy`, `spacy`, `sentence-transformers`, `chromadb`, `polars`, `more-itertools`, and `openai` installed in your environment.
The code was tested with Python 3.14 and the pinned versions in `requirements.txt`. You need Python 3.12 or later.

You can install the dependencies manually, or by running:
You can install the dependencies by running:

```
(venv) $ python -m pip install -r requirements.txt
(venv) $ python -m spacy download en_core_web_lg
```

To run the LLM examples, store your OpenAI API key in a `.env` file in this directory:

```
OPENAI_API_KEY="<your-api-key>"
```
4 changes: 2 additions & 2 deletions embeddings-and-vector-databases-with-chromadb/car_data_etl.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ def prepare_car_reviews_data(
}

# Scan the car reviews dataset(s)
car_reviews = pl.scan_csv(data_path, dtypes=dtypes)
car_reviews = pl.scan_csv(data_path, schema_overrides=dtypes)

# Extract the vehicle title and year as new columns
# Filter on selected years
Expand Down Expand Up @@ -48,7 +48,7 @@ def prepare_car_reviews_data(
"Vehicle_Model",
]
)
.sort(["Vehicle_Model", "Rating"])
.sort(["Vehicle_Model", "Rating"], maintain_order=True)
.collect()
)

Expand Down
7 changes: 4 additions & 3 deletions embeddings-and-vector-databases-with-chromadb/chroma_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,15 @@ def build_chroma_collection(
collection = chroma_client.create_collection(
name=collection_name,
embedding_function=embedding_func,
metadata={"hnsw:space": distance_func_name},
configuration={"hnsw": {"space": distance_func_name}},
)

batch_size = chroma_client.get_max_batch_size()
document_indices = list(range(len(documents)))

for batch in batched(document_indices, 166):
for batch in batched(document_indices, batch_size):
start_idx = batch[0]
end_idx = batch[-1]
end_idx = batch[-1] + 1

collection.add(
ids=ids[start_idx:end_idx],
Expand Down
3 changes: 0 additions & 3 deletions embeddings-and-vector-databases-with-chromadb/config.json

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import chromadb
from car_data_etl import prepare_car_reviews_data
from chroma_utils import build_chroma_collection
from chromadb.utils import embedding_functions

DATA_PATH = "data/archive/*"
CHROMA_PATH = "car_review_embeddings"
Expand All @@ -20,12 +19,9 @@
)

client = chromadb.PersistentClient(CHROMA_PATH)
embedding_func = embedding_functions.SentenceTransformerEmbeddingFunction(
model_name=EMBEDDING_FUNC_NAME
)
collection = client.get_collection(
name=COLLECTION_NAME, embedding_function=embedding_func
)
collection = client.get_collection(name=COLLECTION_NAME)

print(collection.count())

great_reviews = collection.query(
query_texts=[
Expand Down
Original file line number Diff line number Diff line change
@@ -1,41 +1,31 @@
import json
import os

import chromadb
import openai
from chromadb.utils import embedding_functions
from dotenv import load_dotenv
from openai import OpenAI

os.environ["TOKENIZERS_PARALLELISM"] = "false"

DATA_PATH = "data/archive/*"
CHROMA_PATH = "car_review_embeddings"
EMBEDDING_FUNC_NAME = "multi-qa-MiniLM-L6-cos-v1"
COLLECTION_NAME = "car_reviews"
MODEL = "gpt-5.6-luna"

with open("config.json", "r") as json_file:
config_data = json.load(json_file)
load_dotenv()

openai.api_key = config_data.get("openai-secret-key")

client = chromadb.PersistentClient(CHROMA_PATH)
embedding_func = embedding_functions.SentenceTransformerEmbeddingFunction(
model_name=EMBEDDING_FUNC_NAME
)

collection = client.get_collection(
name=COLLECTION_NAME, embedding_function=embedding_func
)
openai_client = OpenAI()
chroma_client = chromadb.PersistentClient(CHROMA_PATH)
collection = chroma_client.get_collection(name=COLLECTION_NAME)

context = """
You are a customer success employee at a large
car dealership. Use the following car reviews
to answer questions: {}
"""
You are a customer success employee at a large
car dealership. Use the following car reviews
to answer questions: {}
"""

question = """
What's the key to great customer satisfaction
based on detailed positive reviews?
"""
What's the key to great customer satisfaction
based on detailed positive reviews?
"""

good_reviews = collection.query(
query_texts=[question],
Expand All @@ -46,45 +36,25 @@

reviews_str = ",".join(good_reviews["documents"][0])

good_review_summaries = openai.ChatCompletion.create(
model="gpt-3.5-turbo",
messages=[
{"role": "system", "content": context.format(reviews_str)},
{"role": "user", "content": question},
],
temperature=0,
n=1,
)

reviews_str = ",".join(good_reviews["documents"][0])

print("Good reviews: ")
print(reviews_str)
print("###########################################")

good_review_summaries = openai.ChatCompletion.create(
model="gpt-3.5-turbo",
messages=[
{"role": "system", "content": context.format(reviews_str)},
{"role": "user", "content": question},
],
temperature=0,
n=1,
good_review_summaries = openai_client.responses.create(
model=MODEL,
instructions=context.format(reviews_str),
input=question,
)

print("AI-Generated summary of good reviews: ")
print(good_review_summaries["choices"][0]["message"]["content"])
print(good_review_summaries.output_text)
print("###########################################")


context = """
You are a customer success employee at a large car dealership.
Use the following car reivews to answer questions: {}
"""
question = """
Which of these poor reviews has the worst implications about
our dealership? Explain why.
"""
Which of these poor reviews has the
worst implications about our dealership?
Explain why.
"""

poor_reviews = collection.query(
query_texts=[question],
Expand All @@ -99,16 +69,12 @@
print(poor_reviews["documents"][0][0])
print("###########################################")

poor_review_analysis = openai.ChatCompletion.create(
model="gpt-3.5-turbo",
messages=[
{"role": "system", "content": context.format(reviews_str)},
{"role": "user", "content": question},
],
temperature=0,
n=1,
poor_review_analysis = openai_client.responses.create(
model=MODEL,
instructions=context.format(reviews_str),
input=question,
)

print("AI-Generated summary of the single worst review: ")
print(poor_review_analysis["choices"][0]["message"]["content"])
print(poor_review_analysis.output_text)
print("###########################################")
99 changes: 7 additions & 92 deletions embeddings-and-vector-databases-with-chromadb/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,92 +1,7 @@
aiohttp==3.8.6
aiosignal==1.3.1
annotated-types==0.6.0
anyio==3.7.1
async-timeout==4.0.3
attrs==23.1.0
backoff==2.2.1
bcrypt==4.0.1
blis==0.7.11
catalogue==2.0.10
certifi==2023.7.22
charset-normalizer==3.3.0
chroma-hnswlib==0.7.3
chromadb==0.4.14
click==8.1.7
cloudpathlib==0.16.0
coloredlogs==15.0.1
confection==0.1.3
cymem==2.0.8
fastapi==0.104.0
filelock==3.12.4
flatbuffers==23.5.26
frozenlist==1.4.0
fsspec==2023.9.2
grpcio==1.59.0
h11==0.14.0
httptools==0.6.1
huggingface-hub==0.17.3
humanfriendly==10.0
idna==3.4
importlib-resources==6.1.0
Jinja2==3.1.2
joblib==1.3.2
langcodes==3.3.0
MarkupSafe==2.1.3
monotonic==1.6
more-itertools==10.1.0
mpmath==1.3.0
multidict==6.0.4
murmurhash==1.0.10
networkx==3.2
nltk==3.8.1
numpy==1.26.1
onnxruntime==1.16.1
openai==0.28.1
overrides==7.4.0
packaging==23.2
Pillow==10.1.0
polars==0.19.9
posthog==3.0.2
preshed==3.0.9
protobuf==4.24.4
pulsar-client==3.3.0
pydantic==2.4.2
pydantic_core==2.10.1
PyPika==0.48.9
python-dateutil==2.8.2
python-dotenv==1.0.0
PyYAML==6.0.1
regex==2023.10.3
requests==2.31.0
safetensors==0.4.0
scikit-learn==1.3.1
scipy==1.11.3
sentence-transformers==2.2.2
sentencepiece==0.1.99
six==1.16.0
smart-open==6.4.0
sniffio==1.3.0
spacy==3.7.2
spacy-legacy==3.0.12
spacy-loggers==1.0.5
srsly==2.4.8
starlette==0.27.0
sympy==1.12
thinc==8.2.1
threadpoolctl==3.2.0
tokenizers==0.14.1
torch==2.1.0
torchvision==0.16.0
tqdm==4.66.1
transformers==4.34.1
typer==0.9.0
typing_extensions==4.8.0
urllib3==2.0.7
uvicorn==0.23.2
uvloop==0.18.0
wasabi==1.1.2
watchfiles==0.21.0
weasel==0.3.3
websockets==11.0.3
yarl==1.9.2
chromadb==1.5.9
numpy==2.5.3
openai==3.14.0
polars==1.44.2
python-dotenv==1.2.3
sentence-transformers==6.0.1
spacy==3.8.16
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import spacy

# Load the medium-size English model
nlp = spacy.load("en_core_web_md")
nlp = spacy.load("en_core_web_lg")

# Get the word vector for the word "dog"
dog_embedding = nlp.vocab["dog"].vector
Expand Down
Loading