Advanced Python on Abel. Dmytro Karpenko Research Infrastructure Services group Department for Scientific Computing USIT, UiO

Transcription

1 Advanced Python on Abel Dmytro Karpenko Research Infrastructure Services group Department for Scientific Computing USIT, UiO

2 Support for large, multi-dimensional arrays and matrices, and a large library of high-level mathematical functions to operate on these arrays Support for scientific computing: optimization, linear algebra, integration, interpolation, statistics, FFT, signal and image processing, etc. Based heavily on NumPy Plotting library, designed especially for use with NumPy, with MatLab-like interface 4/4/16 2

3 NumPy, SciPy, matplotlib Centrally managed on Abel Tight mutual integration Powerful set of tools for data analysis and visualization Usually available on every scientific resource Easy to learn and use The 3 pieces used together can replace MATLAB. 4/4/16 3

4 Getting started on Abel For interactive use: -bash-4.1$ module load python2 -bash-4.1$ module load python2 -bash-4.1$ python -bash-4.1$ python Python (default, Jul , Python (default, Jul , 11:02:23) 11:02:23) [GCC Intel(R) C++ gcc 4.4 mode] [GCC Intel(R) C++ gcc 4.4 mode] on linux2 on linux2 Type "help", "copyright", "credits" or Type "help", "copyright", "credits" or "license" for more information. "license" for more information. >>> import numpy >>> import numpy >>> import scipy >>> import scipy >>> import matplotlib >>> import matplotlib However, you'd rather use matplotlib as >>> >>> 4/4/16 4

5 Numpy General documentation Routines index SciPY Matplotlib 4/4/16 5

6 Numpy arrays >>> import numpy as np >>> import numpy as np >>> cvalues = [25.3, 24.8, 26.9, 23.9] >>> cvalues = [25.3, 24.8, 26.9, 23.9] >>> C = np.array(cvalues) >>> C = np.array(cvalues) >>> print(c) >>> print(c) [ ] [ ] >>> print(c * 9 / ) >>> print(c * 9 / ) [ ] [ ] # Indexing and slicing similar to python lists # Indexing and slicing similar to python lists >>> print C[0] >>> print C[0] >>> print C[1:3] >>> print C[1:3] More straightforward syntax than with lists Considerably faster 4/4/16 6

7 Numpy arrays: advanced addressing and slicing >>> A = np.array([ [3.4, 8.7, 9.9], [1.1, -7.8, -0.7], [4.1, 12.3, 4.8] ]) >>> A = np.array([ [3.4, 8.7, 9.9], [1.1, -7.8, -0.7], [4.1, 12.3, 4.8] ]) >>> print(a[1, 0]) >>> print(a[1, 0]) >>> A = np.array([ >>> A = np.array([... [11,12,13,14,15],... [11,12,13,14,15],... [21,22,23,24,25],... [21,22,23,24,25],... [31,32,33,34,35],... [31,32,33,34,35],... [41,42,43,44,45],... [41,42,43,44,45],... [51,52,53,54,55] ] )... [51,52,53,54,55] ] ) >>> print(a[:3,2:]) >>> print(a[:3,2:]) [[ ] [[ ] [ ] [ ] [ ]] [ ]] 4/4/16 7

8 Numpy arrays: advanced slicing (using step) [start:stop:step] >>> A = np.array([ [ ] >>> A = np.array([ [ ]... [ ]... [ ]... [ ]... [ ]... [ ] ] )... [ ] ] ) >>> print (A[::2, ::3]) >>> print (A[::2, ::3]) [[ 0 3 6] [[ 0 3 6] [ ]] [ ]] >>> print(a[::2, ::3]) >>> print(a[::2, ::3]) [[ 0 3 6] [[ 0 3 6] [ ]] [ ]] 4/4/16 8

9 Numpy arrays: evenly spaced values >>> a = np.arange(1, 10) >>> a = np.arange(1, 10) >>> print(a) >>> print(a) [ ] [ ] >>> x = np.arange(0.5, 10.4, 0.8) >>> x = np.arange(0.5, 10.4, 0.8) >>> print(x) >>> print(x) [ ] [ ] >>> print(np.linspace(1, 10)) >>> print(np.linspace(1, 10)) [ [ ] ] 4/4/16 9

10 Numpy arrays: reshaping >>> x = np.array([ [67, 63, 87], [77, 69, 59], [85, 87, 99], [79, 72, 71], [63, 89, 93], [68, 92, 78]]) >>> x = np.array([ [67, 63, 87], [77, 69, 59], [85, 87, 99], [79, 72, 71], [63, 89, 93], [68, 92, 78]]) >>> print(np.shape(x)) >>> print(np.shape(x)) (6, 3) (6, 3) >>> x.shape = (3, 6) >>> x.shape = (3, 6) >>> print(x) >>> print(x) [[ ] [[ ] [ ] [ ] [ ]] [ ]] >>> X = np.arange(28).reshape(4,7) >>> X = np.arange(28).reshape(4,7) >>> print(x) >>> print(x) [[ ] [[ ] [ ] [ ] [ ] [ ] [ ]] [ ]] 4/4/16 10

11 Scipy statistics >>> from scipy import stats >>> from scipy import stats # Probability density function # Probability density function >>> stats.norm.pdf(0.5) >>> stats.norm.pdf(0.5) # Cumulative distribution function # Cumulative distribution function >>> stats.norm.cdf(0.5) >>> stats.norm.cdf(0.5) # Typical statistics functions # Typical statistics functions >>> norm.mean() >>> norm.mean() >>> norm.median() >>> norm.median() >>> norm.std() >>> norm.std() /4/16 11

12 Scipy statistics >>> a = np.array([1,2,3,4]) >>> a = np.array([1,2,3,4]) >>> b = np.array([10,9,8,7]) >>> b = np.array([10,9,8,7]) # Pearson correlation coefficient # Pearson correlation coefficient >>> stats.pearsonr(a, b) >>> stats.pearsonr(a, b) (-1.0, 0.0) (-1.0, 0.0) # Measure Kolmogorov-Smirnov distance between two samples # Measure Kolmogorov-Smirnov distance between two samples >>> stats.ks_2samp(a,b) >>> stats.ks_2samp(a,b) Ks_2sampResult(statistic=1.0, pvalue= ) Ks_2sampResult(statistic=1.0, pvalue= ) # Maximum Likelihood Estimation # Maximum Likelihood Estimation >>> a=np.array([0.1, 0.2, 0.3, 0.4, 0.5]) >>> a=np.array([0.1, 0.2, 0.3, 0.4, 0.5]) >>> stat.norm.fit(a) >>> stat.norm.fit(a) ( , ) ( , ) 4/4/16 12

13 matplotlib plt.plot([1,2,3,4]) plt.plot([1,2,3,4]) plt.ylabel('some numbers') plt.ylabel('some numbers') 4/4/16 13

14 matplotlib plt.plot([1,2,3,4], [1,4,9,16]) plt.plot([1,2,3,4], [1,4,9,16]) 4/4/16 14

15 matplotlib plt.plot([1,2,3,4], [1,4,9,16], 'ro') plt.plot([1,2,3,4], [1,4,9,16], 'ro') plt.axis([0, 6, 0, 20]) plt.axis([0, 6, 0, 20]) 4/4/16 15

16 matplotlib import numpy as np import numpy as np t = np.arange(0., 5., 0.2) t = np.arange(0., 5., 0.2) plt.plot(t, t, 'r--', t, t**2, 'bs', t, t**3, 'g^') plt.plot(t, t, 'r--', t, t**2, 'bs', t, t**3, 'g^') 4/4/16 16

17 matplotlib import numpy as np import numpy as np mu, sigma = 100, 15 mu, sigma = 100, 15 x = mu + sigma * np.random.randn(10000) x = mu + sigma * np.random.randn(10000) n, bins, patches = plt.hist(x, 50, normed=1, facecolor='g', alpha=0.75) n, bins, patches = plt.hist(x, 50, normed=1, facecolor='g', alpha=0.75) plt.xlabel('smarts') plt.xlabel('smarts') plt.ylabel('probability') plt.ylabel('probability') plt.title('histogram of IQ') plt.title('histogram of IQ') plt.text(60,.025, r'$\mu=100,\ \sigma=15$') plt.text(60,.025, r'$\mu=100,\ \sigma=15$') plt.axis([40, 160, 0, 0.03]) plt.axis([40, 160, 0, 0.03]) plt.grid(true) plt.grid(true) 4/4/16 17

18 matplotlib plt.scatter([x[0] for x in alldata],[x[1] for x in alldata],s=5,marker='+',c='b') plt.scatter([x[0] for x in alldata],[x[1] for x in alldata],s=5,marker='+',c='b') plt.text(200000,6000,"pearson coef = %.3f" % stat.pearsonr([x[0] for x in alldata],[x[1] for x in alldata])[0]) plt.text(200000,6000,"pearson coef = %.3f" % stat.pearsonr([x[0] for x in alldata],[x[1] for x in alldata])[0]) 4/4/16 18

19 matplotlib For non-interactive use: import matplotlib as mpl import matplotlib as mpl mpl.use('agg') mpl.use('agg') now use as usual......but don't forget to save the picture instead of showing x = np.arange(0, 3 * np.pi, 0.1) x = np.arange(0, 3 * np.pi, 0.1) y = np.sin(x) y = np.sin(x) plt.plot(x, y) plt.plot(x, y) plt.savefig("plt_test.png") plt.savefig("plt_test.png") 4/4/16 19

20 Example import numpy as np import numpy as np from scipy.stats import norm from scipy.stats import norm import matplotlib import matplotlib matplotlib.use('agg') matplotlib.use('agg') a = np.array([1, 2, 3]) # Create a rank 1 array a = np.array([1, 2, 3]) # Create a rank 1 array print type(a) # Prints "<type 'numpy.ndarray'>" print type(a) # Prints "<type 'numpy.ndarray'>" print a.shape # Prints "(3,)" print a.shape # Prints "(3,)" print a[0], a[1], a[2] # Prints "1 2 3" print a[0], a[1], a[2] # Prints "1 2 3" a[0] = 5 # Change an element of the array a[0] = 5 # Change an element of the array print a print a print " " print " " print norm.cdf(0) print norm.cdf(0) print " " print " " x = np.arange(0, 3 * np.pi, 0.1) x = np.arange(0, 3 * np.pi, 0.1) y = np.sin(x) y = np.sin(x) # Plot the points using matplotlib # Plot the points using matplotlib plt.plot(x, y) plt.plot(x, y) plt.savefig("plt_test.png") plt.savefig("plt_test.png") print "\ndone" print "\ndone" 4/4/16 20

21 Example #!/bin/bash #!/bin/bash #SBATCH --job-name=advanced_python_test #SBATCH --job-name=advanced_python_test #SBATCH --account=uio #SBATCH --account=uio #SBATCH --time=00:03:00 #SBATCH --time=00:03:00 #SBATCH --mem-per-cpu=4g #SBATCH --mem-per-cpu=4g #SBATCH --mail-type=all #SBATCH --mail-type=all ## Set up job environment: ## Set up job environment: source /cluster/bin/jobsetup source /cluster/bin/jobsetup module purge # clear any inherited modules module purge # clear any inherited modules module load python2 module load python2 set -o errexit # exit on errors set -o errexit # exit on errors ## Copy input files to the work directory: ## Copy input files to the work directory: cp -rf adv_python_test.py $SCRATCH cp -rf adv_python_test.py $SCRATCH ## Do some work: ## Do some work: cd $SCRATCH cd $SCRATCH python pyth.py python pyth.py cp plt_test.png $SUBMITDIR cp plt_test.png $SUBMITDIR 4/4/16 21