simplified the code for multi-threading

multi-threading is now configurable via the commandline


git-svn-id: https://pycam.svn.sourceforge.net/svnroot/pycam/trunk@727 bbaffbd6-741e-11dd-a85d-61de82d9cad9

INSTALL.TXT

@@ -53,6 +53,8 @@ package repository list and run the following:
     apt-get install python-pyode python-opengl
 Afterwards you can remove the new package repository again.
 
+Users of Python 2.5 with multiple CPU cores may want to install the package
+"python-processing". This enables multi-threading for Python before v2.6.
 
 MacOS
 -----

pycam

@@ -178,6 +178,11 @@ def execute(opts, args):
     else:
         log.info("Psyco was disabled via the commandline")
 
+    # initialize multiprocessing
+    if not opts.parallel_processes is None:
+        import pycam.Utils.threading
+        pycam.Utils.threading.init_threading(opts.parallel_processes)
+
     # initialize the progress bar
     progress_styles = {"none": pycam.Gui.Console.ConsoleProgressBar.STYLE_NONE,
             "text": pycam.Gui.Console.ConsoleProgressBar.STYLE_TEXT,
@@ -414,6 +419,11 @@ if __name__ == "__main__":
     group_general.add_option("", "--disable-psyco", dest="disable_psyco",
             default=False, action="store_true", help="disable the Psyco " \
                     + "just-in-time-compiler even when it is available")
+    group_general.add_option("", "--number-of-processes",
+            dest="parallel_processes", default=None, type="int", action="store",
+            help="override the default detection of multiple CPU cores. " \
+                    + "Parallel processing only works with Python 2.6 or " \
+                    + "later.")
     group_general.add_option("-q", "--quiet", dest="quiet",
             default=False, action="store_true", help="show only warnings and " \
             + "errors.")

src/pycam/PathGenerators/ContourFollow.py

@@ -30,6 +30,7 @@ from pycam.PathGenerators import get_free_paths_ode, get_free_paths_triangles
 from pycam.Geometry.utils import epsilon, ceil, sqrt
 from pycam.Geometry import get_bisector, get_angle_pi
 from pycam.Utils import ProgressCounter
+from pycam.Utils.threading import run_in_parallel
 import pycam.Utils.log
 import random
 import math
@@ -286,21 +287,9 @@ class ContourFollow:
         projected_waterlines = []
         triangles = self.model.triangles(minx=minx, miny=miny, maxx=maxx,
                 maxy=maxy)
-        try:
-            # try a multi-threading approach to use multiple CPUs
-            import multiprocessing
-            # use the number of available CPUs
-            pool = multiprocessing.Pool()
-            results_it = pool.imap_unordered(_process_one_triangle,
-                    [(self, t, z) for t in triangles])
-        except ImportError:
-            # use a single-threaded approach
-            def single_generator(triangles):
-                for t in triangles:
-                    # an iterator is expected - thus we use a generator
-                    yield _process_one_triangle((self, t, z))
-            results_it = single_generator(triangles)
-        for result in results_it:
+        results_iter = run_in_parallel(_process_one_triangle,
+                [(self, t, z) for t in triangles], unordered=True)
+        for result in results_iter:
             for edge, shifted_edge in result:
                 waterline_triangles.add(edge, shifted_edge)
             if (not progress_counter is None) \

src/pycam/Utils/threading.py

+# -*- coding: utf-8 -*-
+"""
+$Id$
+
+Copyright 2010 Lars Kruse <devel@sumpfralle.de>
+
+This file is part of PyCAM.
+
+PyCAM is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+PyCAM is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with PyCAM.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import pycam.Utils.log
+
+log = pycam.Utils.log.get_logger()
+
+
+# possible values:
+#   None: not initialized
+#   False: no threading
+#   multiprocessing: the multiprocessing module is impored and enabled
+__multiprocessing = None
+
+# needs to be initialized, if multiprocessing is enabled
+__num_of_processes = None
+
+
+def init_threading(number_of_processes=None):
+    global __multiprocessing, __num_of_processes
+    try:
+        import multiprocessing
+        mp_is_available = True
+    except ImportError:
+        mp_is_available = False
+    if not mp_is_available:
+        __multiprocessing = False
+    else:
+        if number_of_processes is None:
+            # use defaults
+            # don't enable threading for a single cpu
+            if multiprocessing.cpu_count() > 1:
+                __multiprocessing = multiprocessing
+                __num_of_processes = multiprocessing.cpu_count()
+            else:
+                __multiprocessing = False
+        elif number_of_processes < 1:
+            __multiprocessing = False
+        else:
+            __multiprocessing = multiprocessing
+            __num_of_processes = number_of_processes
+    # send the configured state to the logger
+    if __multiprocessing is False:
+        log.info("Disabled multi-threading")
+    else:
+        log.info("Enabled multi-threading with %d parallel processes" % __num_of_processes)
+
+
+def run_in_parallel(func, args, unordered=False):
+    global __multiprocessing, __num_of_processes
+    if __multiprocessing is None:
+        # threading was not configured before
+        init_threading()
+    if __multiprocessing:
+        # use the number of CPUs as the default number of worker threads
+        pool = __multiprocessing.Pool(__num_of_processes)
+        if unordered:
+            imap_func = pool.imap_unordered
+        else:
+            imap_func = pool.imap
+        # Beware: we may not return "pool.imap" or "pool.imap_unordered"
+        # directly. It would somehow loose the focus and just hang infinitely.
+        # Thus we wrap our own generator around it.
+        for result in imap_func(func, args):
+            yield result
+    else:
+        for arg in args:
+            yield func(arg)
+