01-scheduler_default_weights.patch

commit 4c668c47185f2ca2118c91cd016aa3723e679347
Author: Mathieu Gagné <[email protected]>
Date:   Wed Jun 17 14:30:29 2015 -0400

    Add support for scheduler_default_weights
    
    Change-Id: I308bb2b9f9a171d6f72e875d29bc247058c99772

diff --git a/nova/conf/scheduler.py b/nova/conf/scheduler.py
index 6d6dcd0918..e4d46c2176 100644
--- a/nova/conf/scheduler.py
+++ b/nova/conf/scheduler.py
@@ -140,6 +140,53 @@ a different scheduler, this option has no effect.
     exception will be raised.
 """)
 
+host_mgr_avail_mgt_opt = cfg.MultiStrOpt("scheduler_available_weights",
+        default=["nova.scheduler.weights.all_weighers"],
+        help="""
+This is an unordered list of the weight classes the Nova scheduler may apply.
+Only the weighers specified in the 'scheduler_default_weights' option will be
+used, but any weigher appearing in that option must also be included in this
+list.
+
+By default, this is set to all weights that are included with Nova. If you wish
+to change this, replace this with a list of strings, where each element is the
+path to a weight.
+
+This option is only used by the FilterScheduler and its subclasses; if you use
+a different scheduler, this option has no effect.
+
+* Services that use this:
+
+    ``nova-scheduler``
+
+* Related options:
+
+    scheduler_default_weights
+""")
+
+host_mgr_default_wgt_opt = cfg.ListOpt("scheduler_default_weights",
+        default=[
+          "MetricsWeigher",
+          "RAMWeigher",
+          ],
+        help="""
+This option is the list of weighter class names that will be used for weighting
+hosts.
+
+This option is only used by the FilterScheduler and its subclasses; if you use
+a different scheduler, this option has no effect.
+
+* Services that use this:
+
+    ``nova-scheduler``
+
+* Related options:
+
+    All of the weighers in this option *must* be present in the
+    'scheduler_weight_classes' option, or a SchedulerHostWeighterNotFound
+    exception will be raised.
+""")
+
 host_mgr_sched_wgt_cls_opt = cfg.ListOpt("scheduler_weight_classes",
         default=["nova.scheduler.weights.all_weighers"],
         help="""
@@ -765,6 +812,8 @@ default_opts = [host_subset_size_opt,
                use_bm_filters_opt,
                host_mgr_avail_filt_opt,
                host_mgr_default_filt_opt,
+               host_mgr_avail_mgt_opt,
+               host_mgr_default_wgt_opt,
                host_mgr_sched_wgt_cls_opt,
                host_mgr_tracks_inst_chg_opt,
                rpc_sched_topic_opt,
diff --git a/nova/exception.py b/nova/exception.py
index 5006175ef7..c1f036e9c7 100644
--- a/nova/exception.py
+++ b/nova/exception.py
@@ -1219,6 +1219,10 @@ class SchedulerHostFilterNotFound(NotFound):
     msg_fmt = _("Scheduler Host Filter %(filter_name)s could not be found.")
 
 
+class SchedulerHostWeightNotFound(NotFound):
+    msg_fmt = _("Scheduler Host Weight %(weight_name)s could not be found.")
+
+
 class FlavorExtraSpecsNotFound(NotFound):
     msg_fmt = _("Flavor %(flavor_id)s has no extra specs with "
                 "key %(extra_specs_key)s.")
diff --git a/nova/scheduler/host_manager.py b/nova/scheduler/host_manager.py
index 557f4e781b..9d375ce4d9 100644
--- a/nova/scheduler/host_manager.py
+++ b/nova/scheduler/host_manager.py
@@ -343,8 +343,11 @@ class HostManager(object):
         self.default_filters = self._choose_host_filters(self._load_filters())
         self.weight_handler = weights.HostWeightHandler()
         weigher_classes = self.weight_handler.get_matching_classes(
-                CONF.scheduler_weight_classes)
-        self.weighers = [cls() for cls in weigher_classes]
+                CONF.scheduler_available_weights)
+        self.weight_cls_map = {cls.__name__: cls for cls in weigher_classes}
+        self.weight_obj_map = {}
+        self.default_weights = self._choose_host_weights(self._load_weights())
+
         # Dict of aggregates keyed by their ID
         self.aggs_by_id = {}
         # Dict of set of aggregate IDs keyed by the name of the host belonging
@@ -360,6 +363,9 @@ class HostManager(object):
     def _load_filters(self):
         return CONF.scheduler_default_filters
 
+    def _load_weights(self):
+        return CONF.scheduler_default_weights
+
     def _init_aggregates(self):
         elevated = context_module.get_admin_context()
         aggs = objects.AggregateList.get_all(elevated)
@@ -570,10 +576,34 @@ class HostManager(object):
         return self.filter_handler.get_filtered_objects(self.default_filters,
                 hosts, spec_obj, index)
 
-    def get_weighed_hosts(self, hosts, spec_obj):
+    def _choose_host_weights(self, weight_cls_names):
+        if not isinstance(weight_cls_names, (list, tuple)):
+            weight_cls_names = [weight_cls_names]
+
+        good_weights = []
+        bad_weights = []
+        for weight_name in weight_cls_names:
+            if weight_name not in self.weight_obj_map:
+                if weight_name not in self.weight_cls_map:
+                    bad_weights.append(weight_name)
+                    continue
+                weight_cls = self.weight_cls_map[weight_name]
+                self.weight_obj_map[weight_name] = weight_cls()
+            good_weights.append(self.weight_obj_map[weight_name])
+        if bad_weights:
+            msg = ", ".join(bad_weights)
+            raise exception.SchedulerHostWeightNotFound(weight_name=msg)
+        return good_weights
+
+    def get_weighed_hosts(self, hosts, spec_obj,
+                          weight_class_names=None):
         """Weigh the hosts."""
-        return self.weight_handler.get_weighed_objects(self.weighers,
-                hosts, spec_obj)
+        if weight_class_names is None:
+            weights = self.default_weights
+        else:
+            weights = self._choose_host_weights(weight_class_names)
+        return self.weight_handler.get_weighed_objects(
+            weights, hosts, spec_obj)
 
     def get_all_host_states(self, context):
         """Returns a list of HostStates that represents all the hosts

02-ram_weight_multiplier.patch

commit 06b3f18dfd1a4a4aac8710842ee32fb43f5df101
Author: Mathieu Gagné <[email protected]>
Date:   Fri Oct 17 15:52:21 2014 -0400

    Add AggregateRAMWeigher scheduler weight
    
    Add ability to configure ram_weight_multiplier per aggregate.
    
    Change-Id: I58e1050abd41bf7925853440973ddbec147f04fb

diff --git a/nova/scheduler/weights/ram.py b/nova/scheduler/weights/ram.py
index c46ec8458e..9fc3e72c8b 100644
--- a/nova/scheduler/weights/ram.py
+++ b/nova/scheduler/weights/ram.py
@@ -20,10 +20,14 @@ stacking, you can set the 'ram_weight_multiplier' option to a negative
 number and the weighing has the opposite effect of the default.
 """
 
+from oslo_log import log as logging
+
 import nova.conf
+from nova.scheduler.filters import utils
 from nova.scheduler import weights
 
 CONF = nova.conf.CONF
+LOG = logging.getLogger(__name__)
 
 
 class RAMWeigher(weights.BaseHostWeigher):
@@ -36,3 +40,26 @@ class RAMWeigher(weights.BaseHostWeigher):
     def _weigh_object(self, host_state, weight_properties):
         """Higher weights win.  We want spreading to be the default."""
         return host_state.free_ram_mb
+
+
+class AggregateRAMWeigher(weights.BaseHostWeigher):
+    """AggregateRAMWeigher with per-aggregate RAM weight multiplier.
+
+    Fallback to global ram_weight_multiplier if no per-aggregate setting found.
+    """
+
+    def _weigh_object(self, host_state, weight_properties):
+
+        aggregate_vals = utils.aggregate_values_from_key(
+            host_state,
+            'ram_weight_multiplier')
+
+        try:
+            weight = utils.validate_num_values(
+                aggregate_vals,
+                CONF.ram_weight_multiplier, cast_to=float)
+        except ValueError as e:
+            LOG.warning("Could not decode ram_weight_multiplier: '%s'", e)
+            weight = CONF.ram_weight_multiplier
+
+        return host_state.free_ram_mb * weight
diff --git a/nova/tests/unit/scheduler/weights/test_weights_ram.py b/nova/tests/unit/scheduler/weights/test_weights_ram.py
index 2554a7547c..6c491b24c8 100644
--- a/nova/tests/unit/scheduler/weights/test_weights_ram.py
+++ b/nova/tests/unit/scheduler/weights/test_weights_ram.py
@@ -16,6 +16,8 @@
 Tests For Scheduler RAM weights.
 """
 
+import mock
+
 from nova.scheduler import weights
 from nova.scheduler.weights import ram
 from nova import test
@@ -109,3 +111,53 @@ class RamWeigherTestCase(test.NoDBTestCase):
         weighed_host = weights[-1]
         self.assertEqual(0, weighed_host.weight)
         self.assertEqual('negative', weighed_host.obj.host)
+
+    @mock.patch('nova.scheduler.filters.utils.aggregate_values_from_key')
+    def test_aggregate_ram_filter_value_error(self, agg_mock):
+        self.weighers = [ram.AggregateRAMWeigher()]
+        hostinfo_list = self._get_all_hosts()
+        agg_mock.return_value = set(['XXX'])
+
+        weighed_host = self._get_weighed_host(hostinfo_list)
+        self.assertEqual(1.0, weighed_host.weight)
+        self.assertEqual('host4', weighed_host.obj.host)
+
+    @mock.patch('nova.scheduler.filters.utils.aggregate_values_from_key')
+    def test_aggregate_ram_filter_default_value(self, agg_mock):
+        self.weighers = [ram.AggregateRAMWeigher()]
+        hostinfo_list = self._get_all_hosts()
+        agg_mock.return_value = set([])
+
+        weighed_host = self._get_weighed_host(hostinfo_list)
+        self.assertEqual(1.0, weighed_host.weight)
+        self.assertEqual('host4', weighed_host.obj.host)
+
+    @mock.patch('nova.scheduler.filters.utils.aggregate_values_from_key')
+    def test_aggregate_ram_filter(self, agg_mock):
+        self.weighers = [ram.AggregateRAMWeigher()]
+        hostinfo_list = self._get_all_hosts()
+        agg_mock.return_value = set(['3.0'])
+
+        weighed_host = self._get_weighed_host(hostinfo_list)
+        self.assertEqual(1.0, weighed_host.weight)
+        self.assertEqual('host4', weighed_host.obj.host)
+
+    @mock.patch('nova.scheduler.filters.utils.aggregate_values_from_key')
+    def test_aggregate_ram_filter_conflict_values(self, agg_mock):
+        self.weighers = [ram.AggregateRAMWeigher()]
+        hostinfo_list = self._get_all_hosts()
+        agg_mock.return_value = set(['2', '3'])
+
+        weighed_host = self._get_weighed_host(hostinfo_list)
+        self.assertEqual(1.0, weighed_host.weight)
+        self.assertEqual('host4', weighed_host.obj.host)
+
+    @mock.patch('nova.scheduler.filters.utils.aggregate_values_from_key')
+    def test_aggregate_ram_filter_negative(self, agg_mock):
+        self.weighers = [ram.AggregateRAMWeigher()]
+        hostinfo_list = self._get_all_hosts()
+        agg_mock.return_value = set(['-2.0'])
+
+        weighed_host = self._get_weighed_host(hostinfo_list)
+        self.assertEqual(1.0, weighed_host.weight)
+        self.assertEqual('host1', weighed_host.obj.host)