#basic libs

import pandas as pd
import numpy as np
import os
from pathlib import Path

from datetime import datetime, timedelta
import time
from dateutil.relativedelta import relativedelta

import gc
import copy

#additional data processing

import pyarrow.parquet as pq
import pyarrow as pa

from sklearn.preprocessing import StandardScaler, MinMaxScaler


#visualization
import seaborn as sns
import matplotlib.pyplot as plt

#load images
import matplotlib.image as mpimg
import PIL
from PIL import Image




#settings
pd.options.display.max_rows = 100
pd.options.display.max_columns = 100

Image.MAX_IMAGE_PIXELS = None

import warnings
warnings.filterwarnings("ignore")

import pytorch_lightning as pl
random_seed=1234
pl.seed_everything(random_seed)

import os

next(os.walk('/kaggle/input/mayo-clinic-strip-ai'))

train_df = pd.read_csv('/kaggle/input/mayo-clinic-strip-ai/train.csv')

print(train_df.shape)
train_df.head(2)

# check unique number of patients
print(train_df.shape, train_df['patient_id'].nunique())

train_df['patient_id'].value_counts().hist(bins=10)

a = train_df['image_num'].value_counts().sort_index()

t = pd.concat([a, 100*a/train_df.shape[0]], axis=1)
t.columns = ['# samples', '% of samples']
t.index.name = 'image_num'
display(t)
del a, t
gc.collect()

train_df['image_num'].value_counts().plot(kind='bar')

#the center id

a = train_df['center_id'].value_counts().sort_index()

t = pd.concat([a, 100*a/train_df.shape[0]], axis=1)
t.columns = ['# samples', '% of samples']
t.index.name = 'center_id'
display(t)
del a, t
gc.collect()

p_center = pd.pivot_table(train_df, 
               index='patient_id', 
               columns='center_id', 
               values=['image_id'], 
               aggfunc={'image_id':[np.size]}
              )
p_center.columns = [f'{c}' for _, _, c in p_center.columns]
p_center.isna().sum(axis=1).nunique()

p_center.sort_values(by=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11'], inplace=True)

plt.figure(figsize=(12,12))

sns.heatmap(p_center, annot=False, cbar=False)

#label

a = train_df['label'].value_counts().sort_index()

t = pd.concat([a, 100*a/train_df.shape[0]], axis=1)
t.columns = ['# samples', '% of samples']
t.index.name = 'label'
display(t)
del a, t
gc.collect()

train_df['label'].value_counts().plot(kind='bar', figsize=(4,4))

#label

a = train_df[['patient_id', 'label']].drop_duplicates(keep='first')['label'].value_counts().sort_index()

t = pd.concat([a, 100*a/train_df['patient_id'].nunique()], axis=1)
t.columns = ['# unique patients', '% of unique patients']
t.index.name = 'label'
display(t)
del a, t
gc.collect()

p_label = pd.pivot_table(train_df, 
               index='patient_id', 
               columns='label', 
               values=['image_id'], 
               aggfunc={'image_id':[np.size]}
              )
p_label.columns = [f'{c}' for _, _, c in p_label.columns]
# p_label.fillna(value = 0, inplace=True)
p_label['label_cnt'] =2-p_label.isna().sum(axis=1)

p_label['label_cnt'].value_counts()

p_label

p_label['total']= p_label.sum(axis=1)

p_center = pd.pivot_table(train_df[['center_id', 'label', 'patient_id']].drop_duplicates(keep='first'), 
               index='center_id', 
               columns='label', 
               values=['patient_id'], 
               aggfunc={'patient_id':[np.size]}
              )
p_center.columns = [f'{c}' for _, _, c in p_center.columns]

p_center.plot(kind='bar', figsize=(12, 5), 
              title='unique num of patients by target category and center')

%%time
#train file sizes

train_pic_folder = '/kaggle/input/mayo-clinic-strip-ai/train'
train_pics = next(os.walk(train_pic_folder))[2]
#
pic_stats = []
for pic in train_pics:
    p = Path(f'{train_pic_folder}/{pic}')
    img = Image.open(f'{train_pic_folder}/{pic}')
    pic_stats.append([pic.split('.')[0], pic, p.stat().st_size/(1024**2), img.width, img.height, img.mode, img.info['compression'], img.info['dpi'] ])
    img.close()
    del img
    gc.collect()
    
pic_stats_df = pd.DataFrame(data = pic_stats, columns = ['image_id', 'image_name', 'size', 'width', 'height', 'mode', 'compression', 'dpi'])

pic_stats_df.head(2)

pic_stats_df.sort_values(by='size', ascending=False)

pic_stats_df['size'].plot(kind='hist', bins=50, figsize = (8, 5),
                          title='distribution of images by file size (MB)')

pic_stats_df.shape, train_df.shape

train_df = train_df.merge(pic_stats_df, on='image_id', how='left')
train_df.shape

train_df.head(2)

train_df[['label', 'size']].groupby('label').plot(kind='hist', bins=50, figsize = (8, 5),
                          title='distribution of images by file size (MB)')

train_df['center_id'] = train_df['center_id'].astype('category')
train_df[['center_id', 'size']].groupby('center_id').plot(kind='hist', bins=50, figsize = (8, 5),
                          title='distribution of images by file size (MB)')

train_df['ratio'] = train_df['height']/train_df['width']

train_df[['height', 'width', 'ratio']].describe()

for c in ['height', 'width', 'ratio']:
    train_df[[c]].plot(kind='hist', bins=50)

train_df['mode'].value_counts()

train_df['compression'].value_counts()

train_df['dpi'].value_counts()

#find a patient with at 5 small images
train_df[(train_df['size']<500) & (train_df['image_num']==4)]

patient_id='3d10be'
label = 'CE'
train_df[train_df['patient_id']==patient_id]

fig, axes = plt.subplots(nrows=1, ncols=5, figsize=(20, 6))

print(f'patient_id = {patient_id}, label={label}')
for i in range(5):
    img_path = f'/kaggle/input/mayo-clinic-strip-ai/train/{patient_id}_{i}.tif'
    img = Image.open(img_path)
    fac = int(max(img.size)/224)
    h, w = img.size
    
    axes[i].imshow(img.resize((int(h/fac), int(w/fac))))
    axes[i].set_title(f'{patient_id}_{i}')
    
    img.close()
    del img, fac
    gc.collect()

patient_id='91b9d3'
label = 'LAA'
train_df[train_df['patient_id']==patient_id]

fig, axes = plt.subplots(nrows=1, ncols=5, figsize=(20, 6))

print(f'patient_id = {patient_id}, label={label}')
for i in range(5):
    img_path = f'/kaggle/input/mayo-clinic-strip-ai/train/{patient_id}_{i}.tif'
    img = Image.open(img_path)
    fac = int(max(img.size)/224)
    h, w = img.size
    
    axes[i].imshow(img.resize((int(h/fac), int(w/fac))))
    axes[i].set_title(f'{patient_id}_{i}')
    img.close()
    del img, fac
    gc.collect()

%%time
fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(20, 10))
pic_id = '037300_0.tif'
img = Image.open(f"/kaggle/input/mayo-clinic-strip-ai/train/{pic_id}")
print(img.height, img.width)
img.thumbnail((500, 500), resample=Image.Resampling.LANCZOS, reducing_gap=10)
img2 = img.transpose(PIL.Image.Transpose.ROTATE_90)
axes[0].imshow(img)
axes[0].set_title(f'{pic_id} - thumbnail (500, 500)')
axes[1].imshow(img2)
axes[1].set_title(f'{pic_id} - thumbnail (500, 500) rotate 90 degrees')

img.close()
img2.close()

del img, img2
gc.collect()

other_df = pd.read_csv('/kaggle/input/mayo-clinic-strip-ai/other.csv')

print(other_df.shape)
other_df.head(2)

other_df['label'].value_counts()

#train file sizes

other_pic_folder = '/kaggle/input/mayo-clinic-strip-ai/other'
other_pics = next(os.walk(other_pic_folder))[2]
#
pic_stats = []
for pic in other_pics:
    p = Path(f'{other_pic_folder}/{pic}')
    img = Image.open(f'{other_pic_folder}/{pic}')
    pic_stats.append([pic.split('.')[0], pic, p.stat().st_size/(1024**2), img.width, img.height, img.mode, img.info['compression'], img.info['dpi'] ])
    
    img.close()
    del img
    gc.collect()
    
other_pic_stats_df = pd.DataFrame(data = pic_stats, columns = ['image_id', 'image_name', 'size', 'width', 'height', 'mode', 'compression', 'dpi'])

print(other_df.shape, other_pic_stats_df.shape)
other_df = other_df.merge(other_pic_stats_df, on='image_id', how='left')
print(other_df.shape, other_pic_stats_df.shape)
del other_pic_stats_df
gc.collect()

other_df['other_specified'].value_counts()

other_df[other_df['other_specified']=='Hypercoagulable']

%%time
fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(20, 10))
pic_id = '54334d_0.tif'
img = Image.open(f"/kaggle/input/mayo-clinic-strip-ai/other/{pic_id}")
print(img.height, img.width)
img.thumbnail((500, 500), resample=Image.Resampling.LANCZOS, reducing_gap=10)
img2 = img.transpose(PIL.Image.Transpose.ROTATE_90)
axes[0].imshow(img)
axes[0].set_title(f'{pic_id} - thumbnail (500, 500)')
axes[1].imshow(img2)
axes[1].set_title(f'{pic_id} - thumbnail (500, 500) rotate 90 degrees')

img.close()
img2.close()

del img, img2
gc.collect()