Last active
March 23, 2016 16:01
-
-
Save alistairncoles/8e7aeaf64780299a512e to your computer and use it in GitHub Desktop.
Slightly simpler auditor cleanup of unexpected files
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
diff --git a/swift/common/utils.py b/swift/common/utils.py | |
index 210dd9f..8397277 100644 | |
--- a/swift/common/utils.py | |
+++ b/swift/common/utils.py | |
@@ -2124,18 +2124,19 @@ def unlink_older_than(path, mtime): | |
:param path: path to remove file from | |
:param mtime: timestamp of oldest file to keep | |
""" | |
- filepaths = map(functools.partial(os.path.join, path), listdir(path)) | |
- return unlink_paths_older_than(filepaths, mtime) | |
+ return unlink_paths_older_than(path, listdir(path), mtime) | |
-def unlink_paths_older_than(filepaths, mtime): | |
+def unlink_paths_older_than(path_to_dir, filenames, mtime): | |
""" | |
- Remove any files from the given list that that were | |
+ Remove any files from the given directory in the given list that that were | |
last modified before mtime. | |
- :param filepaths: a list of strings, the full paths of files to check | |
+ :param path_to_dir: path to directory from which files are to be removed | |
+ :param filenames: a list of strings, the basenames of files to check | |
:param mtime: timestamp of oldest file to keep | |
""" | |
+ filepaths = map(functools.partial(os.path.join, path_to_dir), filenames) | |
for fpath in filepaths: | |
try: | |
if os.path.getmtime(fpath) < mtime: | |
diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py | |
index ef0253a..ce94755 100644 | |
--- a/swift/obj/auditor.py | |
+++ b/swift/obj/auditor.py | |
@@ -260,12 +260,10 @@ class AuditorWorker(object): | |
# _ondisk_info attr is initialized to None and filled in by open | |
ondisk_info_dict = df._ondisk_info or {} | |
if 'unexpected' in ondisk_info_dict: | |
- is_rsync_tempfile = lambda fpath: RE_RSYNC_TEMPFILE.match( | |
- os.path.basename(fpath)) | |
- rsync_tempfile_paths = filter(is_rsync_tempfile, | |
+ rsync_tempfile_paths = filter(RE_RSYNC_TEMPFILE.match, | |
ondisk_info_dict['unexpected']) | |
mtime = time.time() - self.rsync_tempfile_timeout | |
- unlink_paths_older_than(rsync_tempfile_paths, mtime) | |
+ unlink_paths_older_than(df._datadir, rsync_tempfile_paths, mtime) | |
class ObjectAuditor(Daemon): | |
diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py | |
index 7ea2f25..1454324 100644 | |
--- a/swift/obj/diskfile.py | |
+++ b/swift/obj/diskfile.py | |
@@ -755,10 +755,9 @@ class BaseDiskFileManager(object): | |
file_info['filename'] = afile | |
exts[file_info['ext']].append(file_info) | |
except DiskFileError as e: | |
- file_path = os.path.join(datadir or '', afile) | |
- self.logger.warning('Unexpected file %s: %s', | |
- file_path, e) | |
- results.setdefault('unexpected', []).append(file_path) | |
+ self.logger.warning('Unexpected file %s: %s' % | |
+ (os.path.join(datadir or '', afile), e)) | |
+ results.setdefault('unexpected', []).append(afile) | |
for ext in exts: | |
# For each extension sort files into reverse chronological order. | |
exts[ext] = sorted( |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
diff --git a/test/unit/obj/test_auditor.py b/test/unit/obj/test_auditor.py | |
index 335bf90..9d9fbdc 100644 | |
--- a/test/unit/obj/test_auditor.py | |
+++ b/test/unit/obj/test_auditor.py | |
@@ -313,21 +313,35 @@ class TestAuditor(unittest.TestCase): | |
config_path = os.path.join(self.testdir, 'objserver.conf') | |
stub_config = """ | |
[DEFAULT] | |
- reclaim_age = 604800 | |
+ reclaim_age = 604801 | |
[object-auditor] | |
rsync_tempfile_timeout = auto | |
""" | |
with open(config_path, 'w') as f: | |
f.write(textwrap.dedent(stub_config)) | |
- # the Daemon loader will hand the object-audtior config to the | |
+ # the Daemon loader will hand the object-auditor config to the | |
# auditor who will build the workers from it | |
conf = readconf(config_path, 'object-auditor') | |
auditor_worker = auditor.AuditorWorker(conf, self.logger, | |
self.rcache, self.devices) | |
# if there is no object-replicator section we still have to fall back | |
# to reclaim age because we can't parse the config for that section! | |
- self.assertEqual(auditor_worker.rsync_tempfile_timeout, | |
- 604800) | |
+ self.assertEqual(auditor_worker.rsync_tempfile_timeout, 604801) | |
+ stub_config = """ | |
+ [DEFAULT] | |
+ [object-auditor] | |
+ rsync_tempfile_timeout = auto | |
+ """ | |
+ with open(config_path, 'w') as f: | |
+ f.write(textwrap.dedent(stub_config)) | |
+ # the Daemon loader will hand the object-auditor config to the | |
+ # auditor who will build the workers from it | |
+ conf = readconf(config_path, 'object-auditor') | |
+ auditor_worker = auditor.AuditorWorker(conf, self.logger, | |
+ self.rcache, self.devices) | |
+ # if there is no object-replicator section and no reclaim age option we | |
+ # fall back to one week | |
+ self.assertEqual(auditor_worker.rsync_tempfile_timeout, 604800) | |
stub_config = """ | |
[DEFAULT] | |
reclaim_age = 1209600 | |
@@ -341,7 +355,7 @@ class TestAuditor(unittest.TestCase): | |
auditor_worker = auditor.AuditorWorker(conf, self.logger, | |
self.rcache, self.devices) | |
# if the object-replicator section will parse but does not override | |
- # the default rsync_timeout we assume the default rsync_tiemout value | |
+ # the default rsync_timeout we assume the default rsync_timeout value | |
# and add 15mins | |
self.assertEqual(auditor_worker.rsync_tempfile_timeout, | |
replicator.DEFAULT_RSYNC_TIMEOUT + 900) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Tests need to be updated