Merge remote-tracking branch 'mwilck/master'

Author: mwilck

README.md

@@ -23,7 +23,7 @@ GNU General Public License for more details.**
  * `--ignore-errors`: continue after errors in send/receive. May be useful for
  backing up corrupted file systems. Make sure to check results.
  * `--strategy`: either "parent", "snapshot", "chronological",
- or "generation" (default); see below.
+ "generation" (default), or "bruteforce"; see below.
  * `--toplevel`: don't try to write the target toplevel subvolume, see below.
  * `--btrfs`: set full path to "btrfs" executable.
 
@@ -88,13 +88,14 @@ determine reference subvolumes for each subvolume cloned.
 
 ### Child-parent relationship
 
-Except for the "parent" strategy, the child-parent relationships ("C is a
-snapshot of P") in the target file system will be different from those in the
-source file system. If 3rd party tools rely on a certain parent-child
-relationship, only the "parent" strategy can be used. I tried this with
-**snapper**, and it seemed to work fine with a clone generated with the
-"generation" strategy; apparently it only relies on its own meta data, which
-is preserved in the cloning procedure.
+Except for the "parent" and "bruteforce" strategy, the child-parent
+relationships ("C is a snapshot of P") in the target file system will
+be different from those in the source file system. If 3rd party tools
+rely on a certain parent-child relationship, only the "parent"
+strategy can be used. I tried this with **snapper**, and it seemed to
+work fine with a clone generated with the "generation" strategy;
+apparently it only relies on its own meta data, which is preserved in
+the cloning procedure.
 
 Moreover, file systems will not be cloned in the order of their creation, thus
 when a subvolumeis cloned, we can't be sure that its parent in the filesystem
@@ -134,14 +135,27 @@ where the current fs tree (the default subvolume) has been snapshotted
 several times in the past:
 
  current ---------------------------------\
- | | | |
- snap4 snap3 snap2 snap1
+  | | | |
+  snap4 snap3 snap2 snap1
 
 With "parent" strategy (which was Thomas' original proposal), we'd clone
 "current" first, and after that the snapshots one by one, using "current"
 both as "parent" and "clone source" (["-p" option to btrfs-send][2]) for every
 snapshot.
 
+### "bruteforce" strategy
+
+This strategy is similar to "parent". But it uses every "relative" of
+the subvolume to be cloned as clone source, rather than just the
+direct parent. The set of "relatives" contains all ancestors and all
+descendants of all ancestors. This may lead to a rather large set of
+clone sources, slowing down **btrfs send** operation.
+
+Like "parent", this strategy preserves the child-parent
+relationships. As outlined above, that may be suboptimal for meta data
+cloning. But data cloning should be pretty good with this method, as
+every possible clone source is taken into account.
+
 ### "snapshot" strategy
 
 Obviously, in the picture above, the similarity between snap1 and snap2 will

btrfs-clone

@@ -349,17 +349,81 @@ class SvBaseDir(object):
  if not sv.ro and not opts.dry_run:
  prop_set_ro(newpath, False)
 
-def parents_getter(subvols):
- lookup = { x.uuid: x for x in subvols }
- def _getter(x):
- while x.parent_uuid is not None:
- try:
- x = lookup[x.parent_uuid]
- except KeyError:
- return
+class SubvolSet(object):
+
+ def __init__(self, subvols):
+ self.subvols = subvols
+ self.lookup = { x.uuid: x for x in subvols }
+
+ def parents_getter(self):
+ def _getter(x):
+ while x.parent_uuid is not None:
+ try:
+ x = self.lookup[x.parent_uuid]
+ except KeyError:
+ return
+ else:
+ yield x
+ return _getter
+
+ def get_parents(self, x):
+ return self.parents_getter()(x)
+
+ def get_parent(self, x):
+ if x.parent_uuid is not None and x.parent_uuid in self.lookup:
+ return self.lookup[x.parent_uuid]
+ return None
+
+ def siblings_getter(self):
+ def _getter(x):
+ return (y for y in self.subvols if x.parent_uuid == y.parent_uuid
+ and y.uuid != x.uuid)
+ return _getter
+
+ def get_siblings(self, x):
+ return self.siblings_getter()(x)
+
+ def children_getter(self):
+ def _getter(uuid):
+ return (y for y in self.subvols if y.parent_uuid == uuid)
+ return _getter
+
+ def get_children(self, x):
+ return self.children_getter()(x)
+
+ def descendants_getter(self):
+ def _getter(uuid):
+ children = self.get_children(uuid)
+ for c in children:
+ yield c
+ for x in (y for y in _getter(c.uuid)):
+ yield x
+ return _getter 
+ 
+ def get_descendants(self, x):
+ return self.descendants_getter()(x)
+
+ def relatives_getter(self):
+ def _getter(sv):
+ parents = list(self.get_parents(sv))
+ if parents:
+ grandpa = parents[-1]
  else:
+ grandpa = sv
+ if grandpa.parent_uuid is not None:
+ # oldest ancestor is not in subvols
+ uuid = grandpa.parent_uuid
+ else:
+ uuid = grandpa.uuid
+ if uuid != sv.uuid:
+ yield grandpa
+ for x in (y for y in self.get_descendants(uuid)):
  yield x
- return _getter
+ return _getter
+
+ def get_relatives(self, x):
+ return self.relatives_getter()(x)
+
 
 class Strategy(object):
 
@@ -379,7 +443,8 @@ class Strategy(object):
 
  def prepare_subvols(self):
  self.subvols = get_subvols(self.old)
- self.get_parents = parents_getter(self.subvols)
+ self.svset = SubvolSet(self.subvols)
+ self.get_parents = self.svset.parents_getter()
  self.subvols.sort(key = self.sort_key)
 
  atexit.register(set_all_ro, False, self.subvols, self.old)
@@ -427,6 +492,15 @@ class ParentStrategy(Strategy):
  def _done(self, sv):
  sv.set_ro(False, self.new)
 
+class BruteStrategy(ParentStrategy):
+
+ def send_subvol(self, sv):
+ relatives = (y for y in self.svset.get_relatives(sv) if y.ogen < sv.ogen)
+ flags = self.build_flags(relatives, self.svset.get_parent(sv))
+ do_send_recv(sv.get_path(self.old),
+ os.path.dirname(sv.get_path(self.new)),
+ flags)
+
 class _FlatStrategy(Strategy):
 
  def strategy(self):
@@ -690,14 +764,15 @@ class GenerationStrategy(_FlatStrategy):
  def _done(self, sv):
  self.done = [sv] + self.done
 
+_strategies = {
+ "parent": ParentStrategy,
+ "snapshot": SnapStrategy,
+ "chronological": ChronoStrategy,
+ "generation": GenerationStrategy,
+ "bruteforce": BruteStrategy,
+}
 
 def get_strategy():
- _strategies = {
- "parent": ParentStrategy,
- "snapshot": SnapStrategy,
- "chronological": ChronoStrategy,
- "generation": GenerationStrategy,
- }
  return _strategies[opts.strategy]
 
 def make_args():
@@ -707,8 +782,7 @@ def make_args():
  ps.add_argument("-f", "--force", action='store_true')
  ps.add_argument("-n", "--dry-run", action='store_true')
  ps.add_argument("-s", "--strategy", default="generation",
- choices=["parent", "snapshot", "chronological",
- "generation"])
+ choices=_strategies.keys())
  ps.add_argument("--snap-base")
  ps.add_argument("--no-unshare", action='store_true')
  ps.add_argument("-t", "--toplevel", action='store_false',