time_yield_objects

gistfile1.txt

(swift-3.8.15) (acoles) ~/0dev/openstack/swift{p-misplaced-faster} % git show -q|cat
commit eef67f5319da794b7786ed0613cdbc77889551af
(swift-3.8.15) (acoles) ~/0dev/openstack/swift{p-misplaced-faster} % PYTHONPATH=. python ./time_yield_objects.py 1000000 10
Unable to read test config /etc/swift/test.conf - file not found
yielded 1000000 rows in 1.718s
yielded 1000000 rows in 1.350s
yielded 1000000 rows in 1.317s
yielded 1000000 rows in 1.356s
yielded 1000000 rows in 1.317s
yielded 1000000 rows in 1.535s
yielded 1000000 rows in 1.319s
yielded 1000000 rows in 1.373s
yielded 1000000 rows in 1.321s
yielded 1000000 rows in 1.365s
average rows/sec = 7327785s

(swift-3.8.15) (acoles) ~/0dev/openstack/swift{master,!} % git show -q|cat
commit 5de745c2bc3aa82049424b9c3e8744cda53a4c74
(swift-3.8.15) (acoles) ~/0dev/openstack/swift{master,+} % PYTHONPATH=. python ./time_yield_objects.py 1000000 10
Unable to read test config /etc/swift/test.conf - file not found
yielded 1000000 rows in 2.488s
yielded 1000000 rows in 2.230s
yielded 1000000 rows in 2.203s
yielded 1000000 rows in 2.423s
yielded 1000000 rows in 2.536s
yielded 1000000 rows in 2.182s
yielded 1000000 rows in 2.129s
yielded 1000000 rows in 2.254s
yielded 1000000 rows in 2.403s
yielded 1000000 rows in 2.388s
average rows/sec = 4186907s

time_yield_objects.py

import logging
import random
import sys
import time

from swift.common.utils import Timestamp, ShardRange
from test.unit.container.test_sharder import TestSharder


class Test(TestSharder):
    def time_yield_objects(self, num_objs):
        self.setUp()
        broker = self._make_broker()
        objects = [
            ('o%02d' % i, self.ts_encoded(), 10, 'text/plain', 'etag_a',
             round(random.random()), 0) for i in range(num_objs)]
        for obj in objects:
            broker.put_object(*obj)

        src_range = ShardRange('dont/care', Timestamp.now())
        with self._mock_sharder(conf={}) as sharder:
            sharder.logger = logging.getLogger()
            start = time.time()
            batches = [b for b, _ in
                       sharder.yield_objects(broker, src_range)]
        return time.time() - start


if __name__ == '__main__':
    num_objs = int(sys.argv[1])
    elapsed = Test().time_yield_objects(num_objs)
    print('yielded %d rows in %.3fs' % (num_objs, elapsed))

Modifying the sql query on master makes the search even quicker:

(swift-3.8.15) (acoles) ~/0dev/openstack/swift{master,+} % git diff|cat
diff --git a/swift/container/backend.py b/swift/container/backend.py
index 90f2bf73f..9f8320e2f 100644
--- a/swift/container/backend.py
+++ b/swift/container/backend.py
@@ -1194,7 +1194,7 @@ class ContainerBroker(DatabaseBroker):
                     if conditions:
                         query += 'WHERE ' + ' AND '.join(conditions)
                     tail_query = '''
-                        ORDER BY name %s LIMIT ?
+                        ORDER BY deleted, name %s LIMIT ?
                     ''' % ('DESC' if reverse else '')
                     return query + tail_query, args + [limit - len(results)]

(swift-3.8.15) (acoles) ~/0dev/openstack/swift{master,+} % PYTHONPATH=. python ./time_yield_objects.py 1000000 10
Unable to read test config /etc/swift/test.conf - file not found
yielded 1000000 rows in 0.919s
yielded 1000000 rows in 0.790s
yielded 1000000 rows in 0.816s
yielded 1000000 rows in 0.842s
yielded 1000000 rows in 0.799s
yielded 1000000 rows in 0.703s
yielded 1000000 rows in 0.895s
yielded 1000000 rows in 1.000s
yielded 1000000 rows in 0.992s
yielded 1000000 rows in 0.722s
average rows/sec = 13859567s

sql diff applied to proposed change

(swift-3.8.15) (acoles) ~/0dev/openstack/swift{p-misplaced-faster} % PYTHONPATH=. python ./time_yield_objects.py 1000000 10
Unable to read test config /etc/swift/test.conf - file not found
yielded 1000000 rows in 1.401s
yielded 1000000 rows in 1.330s
yielded 1000000 rows in 1.400s
yielded 1000000 rows in 1.679s
yielded 1000000 rows in 1.311s
yielded 1000000 rows in 1.707s
yielded 1000000 rows in 1.966s
yielded 1000000 rows in 1.791s
yielded 1000000 rows in 1.576s
yielded 1000000 rows in 1.822s
average rows/sec = 5488679s