diff options
author | navanchauhan <navanchauhan@gmail.com> | 2022-05-22 11:42:00 -0600 |
---|---|---|
committer | navanchauhan <navanchauhan@gmail.com> | 2022-05-22 11:42:00 -0600 |
commit | 404c6fb1a3aa8e19db25a5c53098ce4f8a917fd7 (patch) | |
tree | 578b9b21384b889c37b7f9ab2268456e05da8a96 /db2pc.py | |
parent | 801289d123b6e598110a5a1af96605fce9573f7c (diff) |
cleaned with black
Diffstat (limited to 'db2pc.py')
-rw-r--r-- | db2pc.py | 29 |
1 files changed, 17 insertions, 12 deletions
@@ -20,18 +20,23 @@ model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2") batch_size = 32 df = pd.read_sql("Select * from movies", engine) -df["combined_text"] = df["title"] + ": " + df["overview"].fillna('') + " - " + df["tagline"].fillna('') + " Genres:- " + df["genres"].fillna('') +df["combined_text"] = ( + df["title"] + + ": " + + df["overview"].fillna("") + + " - " + + df["tagline"].fillna("") + + " Genres:- " + + df["genres"].fillna("") +) print(f'Length of Combined Text: {len(df["combined_text"].tolist())}') -for x in tqdm(range(0,len(df),batch_size)): - to_send = [] - trakt_ids = df["trakt_id"][x:x+batch_size].tolist() - sentences = df["combined_text"][x:x+batch_size].tolist() - embeddings = model.encode(sentences) - for idx, value in enumerate(trakt_ids): - to_send.append( - ( - str(value), embeddings[idx].tolist() - )) - index.upsert(to_send)
\ No newline at end of file +for x in tqdm(range(0, len(df), batch_size)): + to_send = [] + trakt_ids = df["trakt_id"][x : x + batch_size].tolist() + sentences = df["combined_text"][x : x + batch_size].tolist() + embeddings = model.encode(sentences) + for idx, value in enumerate(trakt_ids): + to_send.append((str(value), embeddings[idx].tolist())) + index.upsert(to_send) |