Changeset 672 for branches

Timestamp:

06/01/13 20:42:22 (3 years ago)

Author:

mmckerns

Message:

updated HPCtut to use pathos and pyina Pools

Location:

branches/HPCtut

Files:

• all_scatter_gather.py (modified) (4 diffs)
• all_scatter_gather2.py (modified) (3 diffs)
• optimize.py (modified) (1 diff)
• optimize_cheby_diffev_map.py (moved) (moved from branches/HPCtut/optimize_cheby_diffev_imap.py) (4 diffs)
• optimize_cheby_powell_map.py (moved) (moved from branches/HPCtut/optimize_cheby_powell_imap.py) (4 diffs)
• optimize_cheby_powell_mpimap.py (modified) (4 diffs)
• optimize_cheby_powell_mpmap.py (modified) (4 diffs)
• optimize_cheby_powell_ppmap.py (modified) (4 diffs)
• optimize_powell.py (modified) (3 diffs)
• optimize_rosen_powell_map.py (moved) (moved from branches/HPCtut/optimize_rosen_powell_imap.py) (4 diffs)

branches/HPCtut/all_scatter_gather.py

@@ -11 +11 @@
 
 import numpy as np
-from pyina.ez_map import ez_map2 as mpi_map  # mpi4py
-from pathos.mp_map import mp_map             # multiprocessing
-from pathos.pp_map import pp_map             # parallelpython
+from pyina.launchers import Mpi as MpiPool
+from pathos.multiprocessing import ProcessingPool
+from pathos.pp import ParallelPythonPool
 nodes = 2; N = 3
 
@@ -36 +36 @@
 # map sin2 to the workers, then print to screen
 print("Running mpi4py on %d cores..." % nodes)
-y = mpi_map(sin2, x, nnodes=nodes)
+y = MpiPool(nodes).map(sin2, x)
 print("Output: %s\n" % np.asarray(y))
 
@@ -42 +42 @@
 # map sin2 to the workers, then print to screen
 print("Running multiprocesing on %d processors..." % nodes)
-y = mp_map(sin2, x, nproc=nodes)
+y = ProcessingPool(nodes).map(sin2, x)
 print("Output: %s\n" % np.asarray(y))
 
@@ -48 +48 @@
 # map sin2 to the workers, then print to screen
 print("Running parallelpython on %d cpus..." % nodes)
-y = pp_map(sin2, x, ncpus=nodes, servers=('mycpu.mydomain.com',))
+y = ParallelPythonPool(nodes).map(sin2, x)
 print("Output: %s\n" % np.asarray(y))
 

branches/HPCtut/all_scatter_gather2.py

@@ -11 +11 @@
 
 import numpy as np
-from pyina.ez_map import ez_map2 as mpi_map  # mpi4py
-from pathos.pp_map import pp_map             # parallelpython
-from pathos.mp_map import mp_map             # multiprocessing
+from pyina.launchers import Mpi as MpiPool
+from pathos.multiprocessing import ProcessingPool
+from pathos.pp import ParallelPythonPool
 nodes = 2; N = 3
 
@@ -36 +36 @@
 # map sin_diff to the workers, then print to screen
 print("Running mpi4py on %d cores..." % nodes)
-y = mpi_map(sin_diff, x, xp, nnodes=nodes)
+y = MpiPool(nodes).map(sin_diff, x, xp)
+print("Output: %s\n" % np.asarray(y))
+
+
+# map sin_diff to the workers, then print to screen
+print("Running multiprocesing on %d processors..." % nodes)
+y = ProcessingPool(nodes).map(sin_diff, x, xp)
 print("Output: %s\n" % np.asarray(y))
 
@@ -42 +48 @@
 # map sin_diff to the workers, then print to screen
 print("Running parallelpython on %d cpus..." % nodes)
-y = pp_map(sin_diff, x, xp, ncpus=nodes)
-print("Output: %s\n" % np.asarray(y))
-
-
-# map sin2 to the workers, then print to screen
-print("Running multiprocesing on %d processors..." % nodes)
-y = mp_map(sin_diff, x, xp, nproc=nodes)
+y = ParallelPythonPool(nodes).map(sin_diff, x, xp)
 print("Output: %s\n" % np.asarray(y))
 

branches/HPCtut/optimize.py

@@ -32 +32 @@
 
     # minimize the function
-    from itertools import imap                   # itertools
-    results = imap(the_solver, model, x0, **kwds)
+    results = map(the_solver, model, x0)
 
     # find the results with the lowest energy

branches/HPCtut/optimize_cheby_diffev_map.py

@@ -2 +2 @@
 """
 Solve Nth-order Chebyshev polynomial coefficients with Differential Evolution.
-Launch optimizers in parallel with itertools map.
+Launch optimizers with python's map.
 
-Requires: development version of mystic
+Requires: development version of mystic, pathos
   http://pypi.python.org/pypi/mystic
+  http://pypi.python.org/pypi/pathos
 """
 
@@ -28 +29 @@
 
     # minimize the function
-    results = mapper(solver, x0, **kwds)
+    results = mapper(nodes).map(solver, x0)
 
     # find the results with the lowest energy
@@ -51 +52 @@
 
 # get the map functions
-from itertools import imap                   # itertools
+from pathos.python import PythonSerial as serial
 
 
@@ -58 +59 @@
     print "Function: %s" % target
     print "Solver: %s" % 'diffev'
-    optimize(diffev_chebyshev, imap, nodes=1, target=target)
+    optimize(diffev_chebyshev, serial, nodes=1, target=target)
 
  

branches/HPCtut/optimize_cheby_powell_map.py

@@ -2 +2 @@
 """
 Solve Nth-order Chebyshev polynomial coefficients with Powell's method.
-Launch optimizers in parallel with itertools map.
+Launch optimizers with python's map.
 
-Requires: development version of mystic
+Requires: development version of mystic, pathos
   http://pypi.python.org/pypi/mystic
+  http://pypi.python.org/pypi/pathos
 """
 
@@ -28 +29 @@
 
     # minimize the function
-    results = mapper(solver, x0, **kwds)
+    results = mapper(nodes).map(solver, x0)
 
     # find the results with the lowest energy
@@ -51 +52 @@
 
 # get the map functions
-from itertools import imap                   # itertools
+from pathos.python import PythonSerial as serial
 
 
@@ -58 +59 @@
     print "Function: %s" % target
     print "Solver: %s" % 'fmin_powell'
-    optimize(powell_chebyshev, imap, nodes=1, target=target)
+    optimize(powell_chebyshev, serial, nodes=1, target=target)
 
  

branches/HPCtut/optimize_cheby_powell_mpimap.py

@@ -2 +2 @@
 """
 Solve Nth-order Chebyshev polynomial coefficients with Powell's method.
-Launch optimizers in parallel with mpi4py map.
+Launch optimizers with mpi4py's map.
 
 Requires: development version of mystic, pyina
@@ -29 +29 @@
 
     # minimize the function
-    results = mapper(solver, x0, **kwds)
+    results = mapper(nodes).map(solver, x0)
 
     # find the results with the lowest energy
@@ -52 +52 @@
 
 # get the map functions
-from pyina.ez_map import ez_map2 as mpi_map  # mpi4py
+from pyina.launchers import Mpi as mpipool
 
 
@@ -59 +59 @@
     print "Function: %s" % target
     print "Solver: %s" % 'fmin_powell'
-    optimize(powell_chebyshev, mpi_map, nodes=10, target=target, nnodes=10)
+    optimize(powell_chebyshev, mpipool, nodes=10, target=target)
 
  

branches/HPCtut/optimize_cheby_powell_mpmap.py

@@ -2 +2 @@
 """
 Solve Nth-order Chebyshev polynomial coefficients with Powell's method.
-Launch optimizers in parallel with multiprocessing map.
+Launch optimizers with multiprocessing's map.
 
 Requires: development version of mystic, pathos
@@ -29 +29 @@
 
     # minimize the function
-    results = mapper(solver, x0, **kwds)
+    results = mapper(nodes).map(solver, x0)
 
     # find the results with the lowest energy
@@ -52 +52 @@
 
 # get the map functions
-from pathos.mp_map import mp_map             # multiprocessing
+from pathos.multiprocessing import ProcessingPool as mppool
 
 
@@ -59 +59 @@
     print "Function: %s" % target
     print "Solver: %s" % 'fmin_powell'
-    optimize(powell_chebyshev, mp_map, nodes=10, target=target, nproc=10)
+    optimize(powell_chebyshev, mppool, nodes=10, target=target)
 
  

branches/HPCtut/optimize_cheby_powell_ppmap.py

@@ -2 +2 @@
 """
 Solve Nth-order Chebyshev polynomial coefficients with Powell's method.
-Launch optimizers in parallel with parallelpython map.
+Launch optimizers with parallelpython's map.
 
 Requires: development version of mystic, pathos
@@ -29 +29 @@
 
     # minimize the function
-    results = mapper(solver, x0, **kwds)
+    results = mapper(nodes).map(solver, x0)
 
     # find the results with the lowest energy
@@ -52 +52 @@
 
 # get the map functions
-from pathos.pp_map import pp_map             # parallelpython
+from pathos.pp import ParallelPythonPool as pppool
 
 
@@ -59 +59 @@
     print "Function: %s" % target
     print "Solver: %s" % 'fmin_powell'
-    optimize(powell_chebyshev, pp_map, nodes=10, target=target, ncpus=10)
+    optimize(powell_chebyshev, pppool, nodes=10, target=target)
 
  

branches/HPCtut/optimize_powell.py

@@ -34 +34 @@
 
     # minimize the function
-    results = mapper(the_solver, model, x0, **kwds)
+    results = mapper(nodes).map(the_solver, model, x0)
 
     # find the results with the lowest energy
@@ -51 +51 @@
 
 # get the map functions
-from itertools import imap                   # itertools
-from pathos.pp_map import pp_map             # parallelpython
-from pathos.mp_map import mp_map             # multiprocessing
-from pyina.ez_map import ez_map2 as mpi_map  # mpi4py
+from pathos.python import PythonSerial as serial
+from pathos.pp import ParallelPythonPool as pppool
+from pathos.multiprocessing import ProcessingPool as mppool
+from pyina.launchers import Mpi as mpipool
 
 
@@ -63 +63 @@
     print "Solver: %s" % 'fmin_powell'
    #NOTE: some of the below should fail, due to how objects are shipped in map
-    optimize(the_solver, imap, nodes=1, target=target)
-   #optimize(the_solver, pp_map, nodes=1, target=target, ncpus=1)
-   #optimize(the_solver, mpi_map, nodes=1, target=target, nnodes=1)
-   #optimize(the_solver, mp_map, nodes=1, target=target, nproc=1)
+    optimize(the_solver, serial, nodes=2, target=target)
+   #optimize(the_solver, mppool, nodes=2, target=target)
+   #optimize(the_solver, pppool, nodes=2, target=target)
+   #optimize(the_solver, mpipool, nodes=2, target=target)  #XXX: Fails
 
  

branches/HPCtut/optimize_rosen_powell_map.py

@@ -2 +2 @@
 """
 Minimize the Rosenbrock function with Powell's method.
-Launch optimizers in parallel with itertools map.
+Launch optimizers with python's map.
 
-Requires: development version of mystic
+Requires: development version of mystic, pathos
   http://pypi.python.org/pypi/mystic
+  http://pypi.python.org/pypi/pathos
 """
 
@@ -28 +29 @@
 
     # minimize the function
-    results = mapper(solver, x0, **kwds)
+    results = mapper(nodes).map(the_solver, model, x0)
 
     # find the results with the lowest energy
@@ -51 +52 @@
 
 # get the map functions
-from itertools import imap                   # itertools
+from pathos.python import PythonSerial as serial
 
 
@@ -58 +59 @@
     print "Function: %s" % target
     print "Solver: %s" % 'fmin_powell'
-    optimize(powell_rosen, imap, nodes=10, target=target)
+    optimize(powell_rosen, serial, nodes=10, target=target)