Pandas - an open source library for fast data analysis, cleaning and preparation

Transcription

1 Pandas - an open source library for fast data analysis, cleaning and preparation In [1]: import numpy as np In [2]: import pandas as pd In [4]: labels = ["a", "b", "c"] In [5]: my_data = [10,20,30] In [6]: arr = np.array(my_data) In [7]: d = {"a":10, "b":20, "c":30} In [9]: print(labels) print(my_data) print(arr) print(d) ['a', 'b', 'c'] [10, 20, 30] [ ] {'a': 10, 'b': 20, 'c': 30} In [10]: pd.series(data = my_data) Out[10]: dtype: int64

2 In [12]: pd.series(data = my_data, index = labels) Out[12]: a 10 b 20 c 30 dtype: int64 In [13]: pd.series(my_data, labels) Out[13]: a 10 b 20 c 30 dtype: int64 In [14]: pd.series(arr) Out[14]: dtype: int32 In [15]: pd.series(arr, labels) Out[15]: a 10 b 20 c 30 dtype: int32 In [16]: d Out[16]: {'a': 10, 'b': 20, 'c': 30} In [17]: pd.series(d) Out[17]: a 10 b 20 c 30 dtype: int64

3 In [18]: pd.series(data = labels) Out[18]: 0 a 1 b 2 c dtype: object In [19]: pd.series(data=[sum,print]) Out[19]: 0 <built-in function sum> 1 <built-in function print> dtype: object In [20]: ser1 = pd.series(data = [1,2,3,4], index = ["USA", "Germany", "USSR", "Japan"]) ser1 Out[20]: USA 1 Germany 2 USSR 3 Japan 4 dtype: int64 In [21]: ser2 = pd.series(data = [1,2,5,4], index = ["USA", "Germany", "Italy", "Japan"]) ser2 Out[21]: USA 1 Germany 2 Italy 5 Japan 4 dtype: int64 In [22]: ser1[0] Out[22]: 1 In [23]: ser1["usa"] Out[23]: 1

4 In [24]: ser2["italy"] Out[24]: 5 In [25]: ser3 = pd.series(data = labels) ser3 Out[25]: 0 a 1 b 2 c dtype: object In [26]: Out[26]: ser3[0] 'a' In [28]: ser1 Out[28]: USA 1 Germany 2 USSR 3 Japan 4 dtype: int64 In [29]: ser2 Out[29]: USA 1 Germany 2 Italy 5 Japan 4 dtype: int64 In [31]: ser1 + ser2 # it does not find a match, so it produces NaN Out[31]: Germany 4.0 Italy NaN Japan 8.0 USA 2.0 USSR NaN dtype: float64

5 Data Frames In [10]: from numpy.random import randn In [32]: np.random.seed(101) In [37]: = pd.dataframe(randn(5,4), ["Row1", "Row2", "Row3", "Row4", "Row5"], ["Col1", "Col2", "Col3", "Col4"]) In [38]: Out[38]: Row Row Row In [39]: ["Col2"] Out[39]: Row Row Row Row Row Name: Col2, dtype: float64 In [40]: Out[40]: type(["col2"]) pandas.core.series.series

6 In [41]: Out[41]: type() pandas.core.frame.dataframe In [44]:.Col4 Out[44]: Row Row Row Row Row Name: Col4, dtype: float64 In [45]: Out[45]: [["Col1","Col2"]] Col1 Col2 Row Row Row Row Row In [46]: ["NewCol"] = ["Col1"] + ["Col2"]

7 In [47]: Out[47]: NewCol Row Row Row In [52]:.drop("NewCol", axis = 1, inplace = True) In [53]: Out[53]: Row Row Row In [57]:.drop("Row5", axis = 0, inplace = True)

8 In [58]: Out[58]: Row Row In [59]:.shape Out[59]: (4, 4) There are two ways to index rows In [61]:.loc["Row1"] Out[61]: Col Col Col Col Name: Row1, dtype: float64 In [62]:.iloc[2] Out[62]: Col Col Col Col Name: Row3, dtype: float64 Subsets of rows and columns

9 Just like Numpy In [63]:.loc["Row1","Col3"] Out[63]: In [65]:.loc[["Row1", "Row2"], ["Col1", "Col3"]] Out[65]: Col1 Col3 Row Row Conditional Selection in data frames In [66]: > 0 Out[66]: Row1 True True False False Row2 False True True True Row3 True True True True Row4 False False False True

10 In [67]: [ > 0] Out[67]: Row NaN NaN Row2 NaN Row4 NaN NaN NaN In [68]: # or bool = > 0 In [69]: Out[69]: bool Row1 True True False False Row2 False True True True Row3 True True True True Row4 False False False True In [70]: [bool] Out[70]: Row NaN NaN Row2 NaN Row4 NaN NaN NaN

11 In [71]: Out[71]: Row Row In [73]: ["Col1"] > 0 Out[73]: Row1 True Row2 False Row3 True Row4 False Name: Col1, dtype: bool In [75]: [["Col1"] > 0] Out[75]: In [76]: Out[76]: Row Row

12 In [77]: [["Col4"] < 0] Out[77]: In [78]: result = [["Col1"] > 0] In [79]: result Out[79]: In [80]: result["col2"] Out[80]: Row Row Name: Col2, dtype: float64 In [81]: # or [["Col1"] > 0]["Col2"] Out[81]: Row Row Name: Col2, dtype: float64 In [82]: [["Col1"] > 0][["Col2", "Col3"]] Out[82]: Col2 Col3 Row Row

13 In [83]: [(["Col1"] > 0) and (["Col2"] > 1)] ValueError Traceback (most recent call last) <ipython-input-83-6c94b9b71d90> in <module>() ----> 1 [(["Col1"] > 0) and (["Col2"] > 1)] D:\Training\PythonforDataScience\lib\site-packages\pandas\core\generic.py in nonzero (self) 953 raise ValueError("The truth value of a {0} is ambiguous. " 954 "Use a.empty, a.bool(), a.item(), a.any() or a.all()." --> 955.format(self. class. name )) bool = nonzero ValueError: The truth value of a Series is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all(). You need to write & for multiple conditions in data frames In [84]: [(["Col1"] > 0) & (["Col2"] > 1)] Out[84]: The same with or In [85]: [(["Col1"] > 0) (["Col2"] > 1)] Out[85]:

14 Index In [86]: Out[86]: Row Row In [87]: Out[87]:.reset_index() index 0 1 Row Row In [91]: new_ind = "CA NY WY OR".split() In [92]: Out[92]: new_ind ['CA', 'NY', 'WY', 'OR'] In [93]: ["States"] = new_ind

15 In [94]: Out[94]: States CA Row NY WY Row OR In [95]: Out[95]:.set_index("States") States CA NY WY OR In [96]: Out[96]: States CA Row NY WY Row OR In [3]: outside = ["G1", "G1", "G1", "G2", "G2", "G2"]

16 In [4]: inside = [1,2,3,1,2,3] In [5]: hier_index = list(zip(outside,inside)) In [6]: hier_index = pd.multiindex.from_tuples(hier_index) In [7]: hier_index Out[7]: MultiIndex(levels=[['G1', 'G2'], [1, 2, 3]], labels=[[0, 0, 0, 1, 1, 1], [0, 1, 2, 0, 1, 2]]) List function with zip function create the following: In [8]: list(zip(outside,inside)) Out[8]: [('G1', 1), ('G1', 2), ('G1', 3), ('G2', 1), ('G2', 2), ('G2', 3)] In [11]: = pd.dataframe(randn(6,2),hier_index,["a","b"]) In [12]: Out[12]: A B G G

17 In [13]: Out[13]:.loc["G1"] A B In [14]:.loc["G1"].loc[1] Out[14]: A B Name: 1, dtype: float64 In [15]: Out[15]:.index.names FrozenList([None, None]) In [16]:.index.names = ["Groups", "Numbers"] In [17]: Out[17]: A B Groups Numbers G G

18 Indexing G2,2,B value, meaning In [19]:.loc["G2"].loc[2].loc["B"] Out[19]: In [20]: # or loc["G1"].loc[3].loc["B"] Out[20]: Cross section In [21]: Out[21]: A B Groups Numbers G G

19 In [22]: Out[22]:.loc["G1"] Numbers A B In [23]: Out[23]:.xs("G1") Numbers A B In [26]:.xs(1,level = "Numbers") # grabbing the ones from both groups Out[26]: A B Groups G G