Transcription

1 1

2 2

3 ##: ********** ## ********** guillimin.hpc.mcgill.ca class## ********** qsub interactive.pbs 3

4 cp -a /software/workshop/cq-formation-advanced-python ~/ cd cq-formation-advanced-python module add git # If on Guillimin git clone -b mcgill \ cd cq-formation-advanced-python 4

5 multiprocessing mpi4py 5

6 6

7 7

8 8

9 // approx_pi.c double approx_pi(int intervals) { double pi = 0.0; # approx_pi.py def approx_pi(intervals): pi = 0.0 int i; for (i = 0; i < intervals; i++) { pi += (4 - ((i % 2) * 8)) / (double)(2 * i + 1); } for i in range(intervals): pi += (4-8 * (i % 2)) / (float)(2 * i + 1) return pi return pi; } 9

10 $ gcc -O2 pi_collect.c approx_pi.c -o pi_collect $./pi_collect $ module add python/3.3.2 $ python3 pi_collect.py approx_pi

11 11

12 $ module add pypy/ $ pypy3 pi_collect.py approx_pi

13 from future import division # only needed for Python 2.x def approx_pi(intervals): pi1 = 4/numpy.arange(1, intervals*2, 4) pi2 = -4/numpy.arange(3, intervals*2, 4) return numpy.sum(pi1) + numpy.sum(pi2) $ python3 pi_collect.py approx_pi_numpy

14 14

15 15

16 /* Example of wrapping approx_pi() with the Python-C-API. */ #include <Python.h> #include "approx_pi.h" static PyObject* approx_pi_func(pyobject* self, PyObject* args) // wrapped approx_pi() { int value; double answer; if (!PyArg_ParseTuple(args, "i", &value)) // parse input, python float to c double return NULL; /* if the above function returns -1, an appropriate Python exception will * have been set, and the function simply returns NULL */ answer =approx_pi(value); /* construct the output from approx_pi, from c double to python float */ return Py_BuildValue("f", answer); } 16

17 /* define functions in module */ static PyMethodDef PiMethods[] = { {"approx_pi", approx_pi_func, METH_VARARGS, "approximate Pi"}, {NULL, NULL, 0, NULL} }; static struct PyModuleDef PiModule = { PyModuleDef_HEAD_INIT, "approx_pi_pyapi", NULL, -1, PiMethods, NULL, NULL, NULL, NULL }; /* module initialization */ PyMODINIT_FUNC PyInit_approx_pi_pyapi(void) { (void) PyModule_Create(&PiModule);} $ python3 setup_approx_pi_pyapi.py build_ext -inplace from distutils.core import setup, Extension # define the extension module module = Extension('approx_pi_pyapi', sources=['approx_pi_pyapi.c', 'approx_pi.c']) setup(ext_modules=[module]) # run the setup 17

18 $ gcc -fpic -shared -O2 approx_pi.c -o approx_pi_ctypes.so # approx_pi_ctypes.py """ Example of wrapping approx_pi using ctypes. """ import ctypes approx_pi_dll = ctypes.cdll.loadlibrary('./approx_pi_ctypes.so') # find and load the library approx_pi_dll.approx_pi.argtypes = [ctypes.c_int] # set the argument type approx_pi_dll.approx_pi.restype = ctypes.c_double # set the return type def approx_pi(arg): ''' Wrapper for approx_pi ''' return approx_pi_dll.approx_pi(arg) 18

19 /* approx_pi_swig.i */ /* Example of wrapping approx_pi using SWIG. */ %module approx_pi_swig %{ /* the resulting C file should be built as a python extension */ #define SWIG_FILE_WITH_INIT /* Includes the header in the wrapper code */ #include "approx_pi.h" %} /* Parse the header file to generate wrappers */ %include "approx_pi.h" 19

20 python3 setup_approx_pi_swig.py build_ext --inplace from distutils.core import setup, Extension approx_pi_module = Extension("_approx_pi", sources= ["approx_pi.c", "approx_pi.i"]) setup(ext_modules=[approx_pi_module]]) approx_pi_swig.py approx_pi_swig_wrap.c _approx_pi_swig*.so 20

21 subroutine approx_pi(intervals, pi) integer, intent(in) :: intervals double precision, intent(out) :: pi integer i pi = 0 do i = 0, intervals - 1 pi = pi + (4 - (mod(i,2) * 8)) / dble(2 * i + 1) enddo end subroutine approx_pi f2py3 -c -m approx_pi_f2py approx_pi.f90 python3 pi_collect.py approx_pi_f2py

22 22

23 .pyx 23

24 approx_pi_cython1.pyx approx_pi.py def approx_pi(intervals): pi = 0.0 for i in range(intervals): pi += (4-8 * (i % 2)) / (float)(2 * i + 1) return pi python3 setup_cython.py build_ext --inplace from distutils.core import setup from Cython.Build import cythonize setup(ext_modules = cythonize("*.pyx")) python3 pi_collect.py approx_pi_cython

25 cdef approx_pi_cython2.pyx def approx_pi(int intervals): cdef double pi cdef int i pi = 0.0 for i in range(intervals): pi += (4-8 * (i % 2)) / (float)(2 * i + 1) return pi python3 setup_cython.py build_ext --inplace python3 pi_collect.py approx_pi_cython

26 approx_pi_cython2.c Pyx_mod_long( pyx_v_i, 2) pyx_v_i % 2. This is because in C, -1% 10=-1 but in Python, -1%10=9. #cython:cdivision=true python3 setup_cython.py build_ext --inplace python3 pi_collect.py approx_pi_cython %load_ext cythonmagic %%cython 26

27 # approx_pi_cython4.pyx cdef extern from "approx_pi.h": double c_approx_pi "approx_pi" (int intervals) # C name: approx_pi, Cython name: c_approx_pi def approx_pi(int intervals): return c_approx_pi(intervals) setup_cython4.py from distutils.core import setup, Extension from Cython.Distutils import build_ext setup(cmdclass={'build_ext': build_ext}, ext_modules=[extension("approx_pi_cython4", sources=["approx_pi_cython4.pyx", "approx_pi.c"])]) python3 setup_cython4.py build_ext --inplace python3 pi_collect.py approx_pi_cython

28 28

29 29

30 30

31 31

32 32

33 multiprocessing 33

34 multiprocessing threading multiprocessing 34

35 from multiprocessing import Pool Pool def prod(values): return values[0] * values[1] if name == ' main ': N = 12 values = [(i + 1, N - i) for i in range(0, N)] print(values) workers = Pool(processes=4) results = workers.map(prod, values) print(results) 35

36 python script.py 36

37 from multiprocessing import Pool import time def prod(values): time.sleep(1) return values[0] * values[1] map_async() if name == ' main ': N = 12 values = [(i + 1, N - i) AsyncResult for i in range(0, N)] print(values) workers = Pool(processes=4) results = workers.map_async(prod, values) print('waiting...') print(results.get(timeout=10)) 37

38 def printres(results): print(results) if name == ' main ': N = 12 values = [(i + 1, N - i) for i in range(0, N)] print(values) workers = Pool(processes=4) results = workers.map_async(prod, values, callback=printres) print('waiting...') workers.close() workers.join() 38

39 Pool([processes[,...]]) processes None processes=multiprocessing.cpu_count() map(func, iterable[,...]) map_async(func, iterable[,...]) AsyncResult close() join() close() 39

40 AsyncResult get([timeout]) wait([timeout]) ready() successful() 40

41 baby-genomic.py edproxy() editdistance() time -p python baby-genomic.py 41

42 Process Process Process(target=fct, args=(arg1,arg2)).start() Pipe Queue Lock 42

43 43

44 44

45 $ ipcluster start -n 4 Pool.map() Engines appear to have started successfully $ bg $ ipython from IPython.parallel import Client client = Client() client.ids [0, 1, 2, 3] def fct(): $ fg return 'hello' client[:].apply_sync(fct) ['hello', 'hello', 'hello', 'hello'] 45

46 python sleep kill ipcluster 46

47 ipython profile create --parallel --profile=name ~/.ipython/profile_name/ipcontroller_config.py c.hubfactory.ip = '*' local01./config_ipcluster.sh local01 47

48 localengines.pbs qsub qsub -N local01 localengines.pbs qstat -u $USER tail -f local01.log Engines appear to have started successfully 48

49 $ ipython --profile=local01 from IPython.parallel import Client client = Client() client.ids with client[:].sync_imports(): import socket %px print(socket.gethostname()) [stdout:0] sw-2r**-n** [stdout:1] sw-2r**-n** [stdout:2] sw-2r**-n** [stdout:3] sw-2r**-n** 49

50 ~/.ipython/profile_name/ipcluster_config.py c.ipclusterengines.engine_launcher_class = 'MPIEngineSetLauncher' mpi01 MPI./config_ipcluster.sh mpi01 MPI 50

51 mpiengines.pbs python openmpi qsub -N mpi01 mpiengines.pbs qstat -u $USER tail -f mpi01.log Engines appear to have started successfully 51

52 $ ipython --profile=mpi01 from IPython.parallel import Client client = Client() with client[:].sync_imports(): import socket, time %px print(socket.gethostname()) def fct(x): time.sleep(1) return x*x client[:].map_sync(fct, range(8)) [0, 1, 4, 9, 16, 25, 36, 49] 52

53 $ cd examples/ $ cat ipclusterex4.py $ ipython --profile=mpi01 from ipclusterex4 import parafct parafct() importing time on engine(s) [0, 1, 4, 9, 16, 25, 36, 49] 53

54 baby-genomic.py ipclustergenomic.py ipclustergenomic.py $ ipython --profile=mpi01 from ipclustergenomic import main main() 54

55 55

56 dest = 1; tag = 54321; MPI_Send( &matrix, count, MPI_INT, dest, tag, MPI_COMM_WORLD ) MPI.COMM_WORLD.Send(matrix, dest=1, tag=54321) 56

57 from mpi4py import MPI Then often use comm = MPI.COMM_WORLD Two variations for most functions: a. all lowercase, e.g. comm.recv() works on general Python objects, using pickle (can be slow) received object (value) returned: matrix = comm.recv(source=0, tag=mpi.any_tag) b. capitalized, e.g. comm.recv() works fast on numpy arrays & other buffers received object given as parameter: comm.recv(matrix, source=0, tag=mpi.any_tag) Specify [matrix, MPI.INT], or [data, count, MPI.INT] if autodetection fails. 57

58 from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.get_rank() size = comm.get_size() print( "%3d/%-3d :-D\n"%rank, size ) mpiexec -n 4 python smiley.py > smiley.out qsub smiley.pbs 58

59 smiley.py smileys.py smileys.py :- :-) :-D :-P smiley.pbs smileys.pbs 59

60 dot_product.py comm.scatter(sendbuf, recvbuf, root=...) comm.reduce comm.reduce comm.gather 60

61 61

62 62