A Beginner’s Guide to Data Analysis with Python & Jupyter Notebook

 Learn how to use Python and Jupyter Notebook

Learn how to use Python and Jupyter Notebook to transform raw datasets into clear insights and interactive charts. A step-by-step beginner's guide to data cleaning, exploration, and visualization

==============================================================

========================================================

import pandas as pd

import numpy as np

import seaborn as sns

import matplotlib.pyplot as plt

 

import os

 

os.listdir(r"E:\Data Analysis work\Datasets")

Output : 

['other-American_B01362.csv',

 'other-Carmel_B00256.csv',

 'other-Dial7_B00887.csv',

 'other-Diplo_B01196.csv',

 'other-Federal_02216.csv',

 'other-FHV-services_jan-aug-2015.csv',

 'other-Firstclass_B01536.csv',

 'other-Highclass_B01717.csv',

 'other-Lyft_B02510.csv',

 'other-Prestige_B01338.csv',

 'other-Skyline_B00111.csv',

 'Uber-Jan-Feb-FOIL.csv',

 'uber-raw-data-apr14.csv',

 'uber-raw-data-aug14.csv',

 'uber-raw-data-janjune-15.csv',

 'uber-raw-data-janjune-15_sample.csv',

 'uber-raw-data-jul14.csv',

 'uber-raw-data-jun14.csv',

 'uber-raw-data-may14.csv',

 'uber-raw-data-sep14.csv']

 

os.listdir(r'E:\Data Analysis work\Datasets')[-7:]

Output: 

['uber-raw-data-aug14.csv',

 'uber-raw-data-janjune-15.csv',

 'uber-raw-data-janjune-15_sample.csv',

 'uber-raw-data-jul14.csv',

 'uber-raw-data-jun14.csv',

 'uber-raw-data-may14.csv',

 'uber-raw-data-sep14.csv']

 

files=os.listdir(r'E:\Data Analysis work\Datasets')[-7:]

files

output: 

['uber-raw-data-aug14.csv',

 'uber-raw-data-janjune-15.csv',

 'uber-raw-data-janjune-15_sample.csv',

 'uber-raw-data-jul14.csv',

 'uber-raw-data-jun14.csv',

 'uber-raw-data-may14.csv',

 'uber-raw-data-sep14.csv']

 

files.remove('uber-raw-data-janjune-15_sample.csv')

files

output:

['uber-raw-data-aug14.csv',

 'uber-raw-data-jul14.csv',

 'uber-raw-data-jun14.csv',

 'uber-raw-data-may14.csv',

 'uber-raw-data-sep14.csv']

 

path=r'E:\Data Analysis work\Datasets'

final=pd.DataFrame()

for file in files:

    df=pd.read_csv(path+"/"+file,encoding='utf-8')

    final=pd.concat([df, final])

 

final.shape

output:

(3969811, 4)

 

df=final.copy()

df.head()

output:

Date/Time	Lat	Lon	Base
0	9/1/2014 0:01:00	40.2201	-74.0021	B02512
1	9/1/2014 0:01:00	40.7500	-74.0027	B02512
2	9/1/2014 0:03:00	40.7559	-73.9864	B02512
3	9/1/2014 0:06:00	40.7450	-73.9889	B02512
4	9/1/2014 0:11:00	40.8145	-73.9444	B02512

 

df.dtypes

output:

Date/Time     object

Lat          float64

Lon          float64

Base          object

dtype: object

 

pd.to_datetime(df['Date/Time'])

output:

0        2014-09-01 00:01:00

1        2014-09-01 00:01:00

2        2014-09-01 00:03:00

3        2014-09-01 00:06:00

4        2014-09-01 00:11:00

                 ...        

829270   2014-08-31 23:55:00

829271   2014-08-31 23:55:00

829272   2014-08-31 23:55:00

829273   2014-08-31 23:59:00

829274   2014-08-31 23:59:00

Name: Date/Time, Length: 3969811, dtype: datetime64[ns]

 

df['Date/Time']=pd.to_datetime(df['Date/Time'], format='%m/%d/%Y %H:%M:%S')

df.dtypes

output:

Date/Time    datetime64[ns]

Lat                 float64

Lon                 float64

Base                 object

dtype: object

 

df.head()

output:

Date/Time	Lat	Lon	Base
0	2014-09-01 00:01:00	40.2201	-74.0021	B02512
1	2014-09-01 00:01:00	40.7500	-74.0027	B02512
2	2014-09-01 00:03:00	40.7559	-73.9864	B02512
3	2014-09-01 00:06:00	40.7450	-73.9889	B02512
4	2014-09-01 00:11:00	40.8145	-73.9444	B02512

 

df['weekday']=df['Date/Time'].dt.day_name()

df['day']=df['Date/Time'].dt.day

df['minute']=df['Date/Time'].dt.minute

df['month']=df['Date/Time'].dt.month

df['hour']=df['Date/Time'].dt.hour

 

df.head()

output:

Date/ Time	Lat	Lon	Base	weekday	day	minute	month	hour
0	2014-09-01 00:01:00	40.2201	-74.0021	B02512	Monday	1	1	9	0
1	2014-09-01 00:01:00	40.7500	-74.0027	B02512	Monday	1	1	9	0
2	2014-09-01 00:03:00	40.7559	-73.9864	B02512	Monday	1	3	9	0
3	2014-09-01 00:06:00	40.7450	-73.9889	B02512	Monday	1	6	9	0
4	2014-09-01 00:11:00	40.8145	-73.9444	B02512	Monday	1	11	9	0

 

 

df.dtypes

output:

Date/Time               datetime64[ns]

Lat                            float64

Lon                            float64

Base                            object

Dispatching_base_num            object

Pickup_date                     object

Affiliated_base_num             object

locationID                     float64

weekday                         object

day                            float64

minute                         float64

month                          float64

hour                           float64

dtype: object

 

df['weekday'].value_counts()

output:

weekday

Thursday     670078

Friday       650836

Wednesday    587857

Tuesday      572604

Saturday     568896

Monday       480611

Sunday       438929

Name: count, dtype: int64

 

%pip install plotly

import plotly.express as px

px.bar()

 

df['weekday'].value_counts()

output:

weekday

Thursday     670078

Friday       650836

Wednesday    587857

Tuesday      572604

Saturday     568896

Monday       480611

Sunday       438929

Name: count, dtype: int64

 

df['weekday'].value_counts().index

output:

Index(['Thursday', 'Friday', 'Wednesday', 'Tuesday', 'Saturday', 'Monday',

       'Sunday'],

      dtype='object', name='weekday')

 

px.bar(x=df['weekday'].value_counts().index,

       y=df['weekday'].value_counts()

      )

Output:

Crate chart

 

plt.hist(df['hour'])

output:

 

df['month'].unique()

for i in df['month'].unique():

    plt.subplot(3,2,i+1)

 

    (only example )

 

for i,month in enumerate(df['month'].unique()):

    print(i)

    print(month)

 

 

plt.figure(figsize=(40,20))

for i,month in enumerate(df['month'].unique()):

    plt.subplot(3,2,i+1)

    df[df['month']==month]['hour'].hist()

 

 

Video  : 8