blob: 48d3c4af778ec49defff55f4fc6513bece600d56 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
|
import os
import pinecone
from database import *
import pandas as pd
from sentence_transformers import SentenceTransformer
from tqdm import tqdm
database_url = "sqlite:///jlm.db"
engine, Session = init_db_stuff(database_url)
PINECONE_KEY = os.getenv("PINECONE_API_DEFAULT")
pinecone.init(api_key=PINECONE_KEY, environment="us-west1-gcp")
index = pinecone.Index("movies")
model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")
batch_size = 32
df = pd.read_sql("Select * from movies", engine)
df["combined_text"] = (
df["title"]
+ ": "
+ df["overview"].fillna("")
+ " - "
+ df["tagline"].fillna("")
+ " Genres:- "
+ df["genres"].fillna("")
)
print(f'Length of Combined Text: {len(df["combined_text"].tolist())}')
for x in tqdm(range(0, len(df), batch_size)):
to_send = []
trakt_ids = df["trakt_id"][x : x + batch_size].tolist()
sentences = df["combined_text"][x : x + batch_size].tolist()
embeddings = model.encode(sentences)
for idx, value in enumerate(trakt_ids):
to_send.append((str(value), embeddings[idx].tolist()))
index.upsert(to_send)
|
