!ls

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import json
import glob
import lzma
from sklearn.linear_model import LinearRegression
from datetime import timedelta, datetime

global_df = []

def to_xlsx(file):
    info = {}

    with open(file) as f:
        data = json.load(f)
        try:
            info["post_type_name"] = data["node"]["__typename"]
        except KeyError:
            info["post_type_name"] = "null"
        try:
            info["post_text"] = data["node"]["edge_media_to_caption"]["edges"][0]["node"]["text"]
        except (IndexError, KeyError):
            info["post_text"] = "null"
        try:
            info["account_name"] = data["node"]["owner"]["id"]
        except KeyError:
            info["account_name"] = "null"
        try:
            info["shortcode"] = "https://www.instagram.com/p/"+data["node"]["shortcode"]
        except KeyError:
            info["shortcode"] = 0
        try:
            dt = data["node"]["taken_at_timestamp"]
            info["timestamp"] = datetime.fromtimestamp(dt)
        except KeyError:
            info["timestamp"] = 0
        try:
            info["like_count"] = data["node"]["edge_media_preview_like"]["count"]
        except KeyError:
            info["like_count"] = 0
        try:
            info["comment_count"] = data["node"]["edge_media_to_comment"]["count"]
        except KeyError:
            info["comment_count"] = 0
        try:
            info["video_view_count"] = data["node"]["video_view_count"]
        except (IndexError, KeyError):
            info["video_view_count"] = 0
        try:
            info["comments_disabled"] = data["node"]["comments_disabled"]
        except KeyError:
            info["comments_disabled"] = "null"

        # AFEGITS

        try:
            info["full_name"] = data["node"]["owner"]["full_name"]
        except KeyError:
            info["full_name"] = "null"
        try:
            info["is_professional_account"] = data["node"]["owner"]["is_professional_account"]
        except KeyError:
            info["is_professional_account"] = "null"
        try:
            info["is_business_account"] = data["node"]["owner"]["is_business_account"]
        except KeyError:
            info["is_business_account"] = "null"
        try:
            info["is_verified"] = data["node"]["owner"]["is_verified"]
        except KeyError:
            info["is_verified"] = "null"
        try:
            info["is_video"] = data["node"]["is_video"]
        except KeyError:
            info["is_video"] = "null"
        try:
            info["category_name"] = data["node"]["owner"]["category_name"]
        except KeyError:
            info["category_name"] = "null"

        return info # returning the dict with all data

json_files = glob.glob("./curacingteam/*.json.xz") # Path To JSON FILES
ammount = len(json_files)
print(f"Found {ammount} posts")

for json_file in json_files:
    with open("./temp.json","w") as f:
        f.write(lzma.open(json_file,mode='rt',encoding='utf-8').read())
    info = to_xlsx('./temp.json')  # This is the main function
    
    df = pd.DataFrame({
        "post_date": info["timestamp"],
        "account_id": info["account_name"],
        "full_name": info["full_name"],
        "text": info["post_text"],
        "post_shortcode":  info["shortcode"],
        "like_count": info["like_count"],
        "comment_count": info["comment_count"],
        "is_video": info["is_video"],
        "video_view_count": info["video_view_count"],
        "comments_policy": info["comments_disabled"],
        "is_professional": info["is_professional_account"],
        "is_business": info["is_business_account"],
        "is_verified": info["is_verified"],
        "person_category": info["category_name"],
        "post_type": info["post_type_name"]
    }, index=[1])

    global_df.append(df)

final = pd.concat(global_df)
#final.to_csv("filename.csv", index=False) # Your Filename
print("Converted...")

final

# Input dictionary
dic = {0: 1664, 7: 1661, 14: 1657, 21: 1639, 28: 1589, 30: 1568, 37: 1538, 44: 1532, 51: 1534, 58: 1529, 65: 1522, 72: 1515, 79: 1511, 90: 1482, 2636: 10}

# Extract keys and values from the dictionary
keys = np.array(list(dic.keys())).reshape(-1, 1)
values = np.array(list(dic.values()))

# Create a LinearRegression model
model = LinearRegression()

# Fit the model to the data
model.fit(keys, values)

# Predict the number of followers for a new day
new_day = 0
predicted_followers = model.predict([[new_day]])

print("Predicted number of followers for day", new_day, ":", predicted_followers[0])

df = final

# Exclude rows with 'post_date' of type int
df = df[~df['post_date'].apply(lambda x: isinstance(x, int))]

# Then convert to datetime
df['post_date'] = pd.to_datetime(df['post_date'])

# Let's assume 'today' is the latest date in the dataframe
#today = df['post_date'].max()
today = datetime.strptime('15/06/23 15:25:00', '%d/%m/%y %H:%M:%S')

def magic_function(row):
    
    days = (today - row['post_date']).days
    return model.predict([[days]])[0]

"""# Convert the date to the number of days before today
df['days_before_today'] = (today - df['post_date']).dt.days

# Convert the dictionary to a DataFrame for interpolation
dic_df = pd.DataFrame(list(dic.items()), columns=['days_before_today', 'followers'])

# Sort the DataFrame based on 'days_before_today'
dic_df = dic_df.sort_values('days_before_today')

# Interpolate the followers for the days not present in the dictionary
dic_df = dic_df.set_index('days_before_today').reindex(range(dic_df.days_before_today.min(), dic_df.days_before_today.max() + 1)).interpolate().reset_index()

# Merge the two DataFrames on 'days_before_today'
df = pd.merge(df, dic_df, on='days_before_today', how='left')

df['followers'] = df['followers'].replace(np.nan, 1000)"""

df['followers'] = df.apply(magic_function, axis=1) #df.apply(lambda row : model.predict([[(today - row['post_date']).dt.days]])[0])

# Rename the column to 'followers_at_time'
df = df.rename(columns={'followers': 'followers_at_time'})

# Calculate the engagement_percentage
df['engagement_percentage'] = (df['like_count'] + df['comment_count'] + df['video_view_count']) / df['followers_at_time'] * 100

processed_df = df

f, axs = plt.subplots(1, 2, figsize=(8, 4), gridspec_kw=dict(width_ratios=[4, 3]))
sns.scatterplot(data=processed_df, x="post_date", y="engagement_percentage", hue="post_type", ax=axs[0])
sns.histplot(data=processed_df, x="post_type", hue="post_type", shrink=.8, alpha=.8, legend=False, ax=axs[1])
f.tight_layout()

f, axs = plt.subplots(1, 2, figsize=(8, 4), gridspec_kw=dict(width_ratios=[4, 3]))
sns.scatterplot(data=processed_df, x="post_date", y="like_count", hue="post_type", ax=axs[0])
sns.histplot(data=processed_df, x="post_type", hue="post_type", shrink=.8, alpha=.8, legend=False, ax=axs[1])
f.tight_layout()

f, axs = plt.subplots(1, 2, figsize=(8, 4), gridspec_kw=dict(width_ratios=[4, 3]))
sns.scatterplot(data=processed_df, x="post_date", y="comment_count", hue="post_type", ax=axs[0])
sns.histplot(data=processed_df, x="post_type", hue="post_type", shrink=.8, alpha=.8, legend=False, ax=axs[1])
f.tight_layout()