aboutsummaryrefslogtreecommitdiff
path: root/db2pc.py
diff options
context:
space:
mode:
authornavanchauhan <navanchauhan@gmail.com>2022-05-22 11:42:00 -0600
committernavanchauhan <navanchauhan@gmail.com>2022-05-22 11:42:00 -0600
commit404c6fb1a3aa8e19db25a5c53098ce4f8a917fd7 (patch)
tree578b9b21384b889c37b7f9ab2268456e05da8a96 /db2pc.py
parent801289d123b6e598110a5a1af96605fce9573f7c (diff)
cleaned with black
Diffstat (limited to 'db2pc.py')
-rw-r--r--db2pc.py29
1 files changed, 17 insertions, 12 deletions
diff --git a/db2pc.py b/db2pc.py
index 4acddcb..48d3c4a 100644
--- a/db2pc.py
+++ b/db2pc.py
@@ -20,18 +20,23 @@ model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")
batch_size = 32
df = pd.read_sql("Select * from movies", engine)
-df["combined_text"] = df["title"] + ": " + df["overview"].fillna('') + " - " + df["tagline"].fillna('') + " Genres:- " + df["genres"].fillna('')
+df["combined_text"] = (
+ df["title"]
+ + ": "
+ + df["overview"].fillna("")
+ + " - "
+ + df["tagline"].fillna("")
+ + " Genres:- "
+ + df["genres"].fillna("")
+)
print(f'Length of Combined Text: {len(df["combined_text"].tolist())}')
-for x in tqdm(range(0,len(df),batch_size)):
- to_send = []
- trakt_ids = df["trakt_id"][x:x+batch_size].tolist()
- sentences = df["combined_text"][x:x+batch_size].tolist()
- embeddings = model.encode(sentences)
- for idx, value in enumerate(trakt_ids):
- to_send.append(
- (
- str(value), embeddings[idx].tolist()
- ))
- index.upsert(to_send) \ No newline at end of file
+for x in tqdm(range(0, len(df), batch_size)):
+ to_send = []
+ trakt_ids = df["trakt_id"][x : x + batch_size].tolist()
+ sentences = df["combined_text"][x : x + batch_size].tolist()
+ embeddings = model.encode(sentences)
+ for idx, value in enumerate(trakt_ids):
+ to_send.append((str(value), embeddings[idx].tolist()))
+ index.upsert(to_send)