From ea3ea772562df81beb4df20b85fd5d49a5878aa3 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 31 Jan 2017 14:38:36 +0100 Subject: [PATCH] Correctly remove source duplicates. Duplicate sources must be detected from their classes. So add a optional arguments key_function to remove_duplicates to use a custom key to compare elems and use it to remove duplicate sources using their classes. --- kiwix-build.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/kiwix-build.py b/kiwix-build.py index 5593386..a4c43fc 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -23,12 +23,15 @@ class Defaultdict(defaultdict): def __getattr__(self, name): return self[name] -def remove_duplicates(iterable): +def remove_duplicates(iterable, key_function=None): seen = set() + if key_function is None: + key_function = lambda e:e for elem in iterable: - if elem in seen: + key = key_function(elem) + if key in seen: continue - seen.add(elem) + seen.add(key) yield elem def get_sha256(path): @@ -778,7 +781,8 @@ class Builder: yield targetName def prepare_sources(self): - sources = remove_duplicates(dep.source for dep in self.targets.values()) + sources = (dep.source for dep in self.targets.values()) + sources = remove_duplicates(sources, lambda s:s.__class__) for source in sources: print("prepare sources {} :".format(source.name)) source.prepare()