jakob/myshellconfig
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity
myshellconfig/bin/git-filter-repo

4005 lines
162 KiB
Python
Executable file
Raw Blame History

#!/usr/bin/env python3
"""
git-filter-repo filters git repositories, similar to git filter-branch, BFG
repo cleaner, and others. The basic idea is that it works by running
git fast-export <options> | filter | git fast-import <options>
where this program not only launches the whole pipeline but also serves as
the 'filter' in the middle. It does a few additional things on top as well
in order to make it into a well-rounded filtering tool.
git-filter-repo can also be used as a library for more involved filtering
operations; however:
***** API BACKWARD COMPATIBILITY CAVEAT *****
Programs using git-filter-repo as a library can reach pretty far into its
internals, but I am not prepared to guarantee backward compatibility of
all APIs. I suspect changes will be rare, but I reserve the right to
change any API. Since it is assumed that repository filtering is
something one would do very rarely, and in particular that it's a
one-shot operation, this should not be a problem in practice for anyone.
However, if you want to re-use a program you have written that uses
git-filter-repo as a library (or makes use of one of its --*-callback
arguments), you should either make sure you are using the same version of
git and git-filter-repo, or make sure to re-test it.
If there are particular pieces of the API you are concerned about, and
there is not already a testcase for it in t9391-lib-usage.sh or
t9392-python-callback.sh, please contribute a testcase. That will not
prevent me from changing the API, but it will allow you to look at the
history of a testcase to see whether and how the API changed.
***** END API BACKWARD COMPATIBILITY CAVEAT *****
"""
import argparse
import collections
import fnmatch
import gettext
import io
import os
import platform
import re
import shutil
import subprocess
import sys
import time
import textwrap
from datetime import tzinfo, timedelta, datetime
__all__ = ["Blob", "Reset", "FileChange", "Commit", "Tag", "Progress",
"Checkpoint", "FastExportParser", "ProgressWriter",
"string_to_date", "date_to_string",
"record_id_rename", "GitUtils", "FilteringOptions", "RepoFilter"]
deleted_hash = b'0'*40
write_marks = True
date_format_permissive = True
def gettext_poison(msg):
if "GIT_TEST_GETTEXT_POISON" in os.environ: # pragma: no cover
return "# GETTEXT POISON #"
return gettext.gettext(msg)
_ = gettext_poison
def setup_gettext():
TEXTDOMAIN="git-filter-repo"
podir = os.environ.get("GIT_TEXTDOMAINDIR") or "@@LOCALEDIR@@"
if not os.path.isdir(podir): # pragma: no cover
podir = None # Python has its own fallback; use that
## This looks like the most straightforward translation of the relevant
## code in git.git:gettext.c and git.git:perl/Git/I18n.pm:
#import locale
#locale.setlocale(locale.LC_MESSAGES, "");
#locale.setlocale(locale.LC_TIME, "");
#locale.textdomain(TEXTDOMAIN);
#locale.bindtextdomain(TEXTDOMAIN, podir);
## but the python docs suggest using the gettext module (which doesn't
## have setlocale()) instead, so:
gettext.textdomain(TEXTDOMAIN);
gettext.bindtextdomain(TEXTDOMAIN, podir);
def _timedelta_to_seconds(delta):
"""
Converts timedelta to seconds
"""
offset = delta.days*86400 + delta.seconds + (delta.microseconds+0.0)/1000000
return round(offset)
class FixedTimeZone(tzinfo):
"""
Fixed offset in minutes east from UTC.
"""
tz_re = re.compile(br'^([-+]?)(\d\d)(\d\d)$')
def __init__(self, offset_string):
tzinfo.__init__(self)
sign, hh, mm = FixedTimeZone.tz_re.match(offset_string).groups()
factor = -1 if (sign and sign == b'-') else 1
self._offset = timedelta(minutes = factor*(60*int(hh) + int(mm)))
self._offset_string = offset_string
def utcoffset(self, dt):
return self._offset
def tzname(self, dt):
return self._offset_string
def dst(self, dt):
return timedelta(0)
def string_to_date(datestring):
(unix_timestamp, tz_offset) = datestring.split()
return datetime.fromtimestamp(int(unix_timestamp),
FixedTimeZone(tz_offset))
def date_to_string(dateobj):
epoch = datetime.fromtimestamp(0, dateobj.tzinfo)
return(b'%d %s' % (int(_timedelta_to_seconds(dateobj - epoch)),
dateobj.tzinfo.tzname(0)))
def decode(bytestr):
'Try to convert bytestr to utf-8 for outputting as an error message.'
return bytestr.decode('utf-8', 'backslashreplace')
def glob_to_regex(glob_bytestr):
'Translate glob_bytestr into a regex on bytestrings'
# fnmatch.translate is idiotic and won't accept bytestrings
if (decode(glob_bytestr).encode() != glob_bytestr): # pragma: no cover
raise SystemExit(_("Error: Cannot handle glob %s").format(glob_bytestr))
# Create regex operating on string
regex = fnmatch.translate(decode(glob_bytestr))
# FIXME: This is an ugly hack...
# fnmatch.translate tries to do multi-line matching and wants the glob to
# match up to the end of the input, which isn't relevant for us, so we
# have to modify the regex. fnmatch.translate has used different regex
# constructs to achieve this with different python versions, so we have
# to check for each of them and then fix it up. It would be much better
# if fnmatch.translate could just take some flags to allow us to specify
# what we want rather than employing this hackery, but since it
# doesn't...
if regex.endswith(r'\Z(?ms)'): # pragma: no cover
regex = regex[0:-7]
elif regex.startswith(r'(?s:') and regex.endswith(r')\Z'): # pragma: no cover
regex = regex[4:-3]
# Finally, convert back to regex operating on bytestr
return regex.encode()
class PathQuoting:
_unescape = {b'a': b'\a',
b'b': b'\b',
b'f': b'\f',
b'n': b'\n',
b'r': b'\r',
b't': b'\t',
b'v': b'\v',
b'"': b'"',
b'\\':b'\\'}
_unescape_re = re.compile(br'\\([a-z"\\]|[0-9]{3})')
_escape = [bytes([x]) for x in range(127)]+[
b'\\'+bytes(ord(c) for c in oct(x)[2:]) for x in range(127,256)]
_reverse = dict(map(reversed, _unescape.items()))
for x in _reverse:
_escape[ord(x)] = b'\\'+_reverse[x]
_special_chars = [len(x) > 1 for x in _escape]
@staticmethod
def unescape_sequence(orig):
seq = orig.group(1)
return PathQuoting._unescape[seq] if len(seq) == 1 else bytes([int(seq, 8)])
@staticmethod
def dequote(quoted_string):
if quoted_string.startswith(b'"'):
assert quoted_string.endswith(b'"')
return PathQuoting._unescape_re.sub(PathQuoting.unescape_sequence,
quoted_string[1:-1])
return quoted_string
@staticmethod
def enquote(unquoted_string):
# Option 1: Quoting when fast-export would:
# pqsc = PathQuoting._special_chars
# if any(pqsc[x] for x in set(unquoted_string)):
# Option 2, perf hack: do minimal amount of quoting required by fast-import
if unquoted_string.startswith(b'"') or b'\n' in unquoted_string:
pqe = PathQuoting._escape
return b'"' + b''.join(pqe[x] for x in unquoted_string) + b'"'
return unquoted_string
class AncestryGraph(object):
"""
A class that maintains a direct acycle graph of commits for the purpose of
determining if one commit is the ancestor of another.
"""
def __init__(self):
self.cur_value = 0
# A mapping from the external identifers given to us to the simple integers
# we use in self.graph
self.value = {}
# A tuple of (depth, list-of-ancestors). Values and keys in this graph are
# all integers from the self.value dict. The depth of a commit is one more
# than the max depth of any of its ancestors.
self.graph = {}
# Cached results from previous calls to is_ancestor().
self._cached_is_ancestor = {}
def record_external_commits(self, external_commits):
"""
Record in graph that each commit in external_commits exists, and is
treated as a root commit with no parents.
"""
for c in external_commits:
if c not in self.value:
self.cur_value += 1
self.value[c] = self.cur_value
self.graph[self.cur_value] = (1, [])
def add_commit_and_parents(self, commit, parents):
"""
Record in graph that commit has the given parents. parents _MUST_ have
been first recorded. commit _MUST_ not have been recorded yet.
"""
assert all(p in self.value for p in parents)
assert commit not in self.value
# Get values for commit and parents
self.cur_value += 1
self.value[commit] = self.cur_value
graph_parents = [self.value[x] for x in parents]
# Determine depth for commit, then insert the info into the graph
depth = 1
if parents:
depth += max(self.graph[p][0] for p in graph_parents)
self.graph[self.cur_value] = (depth, graph_parents)
def is_ancestor(self, possible_ancestor, check):
"""
Return whether possible_ancestor is an ancestor of check
"""
a, b = self.value[possible_ancestor], self.value[check]
original_pair = (a,b)
a_depth = self.graph[a][0]
ancestors = [b]
visited = set()
while ancestors:
ancestor = ancestors.pop()
prev_pair = (a, ancestor)
if prev_pair in self._cached_is_ancestor:
if not self._cached_is_ancestor[prev_pair]:
continue
self._cached_is_ancestor[original_pair] = True
return True
if ancestor in visited:
continue
visited.add(ancestor)
depth, more_ancestors = self.graph[ancestor]
if ancestor == a:
self._cached_is_ancestor[original_pair] = True
return True
elif depth <= a_depth:
continue
ancestors.extend(more_ancestors)
self._cached_is_ancestor[original_pair] = False
return False
class MailmapInfo(object):
def __init__(self, filename):
self.changes = {}
self._parse_file(filename)
def _parse_file(self, filename):
name_and_email_re = re.compile(br'(.*?)\s*<([^>]*)>\s*')
comment_re = re.compile(br'\s*#.*')
if not os.access(filename, os.R_OK):
raise SystemExit(_("Cannot read %s") % decode(filename))
with open(filename, 'br') as f:
count = 0
for line in f:
count += 1
err = "Unparseable mailmap file: line #{} is bad: {}".format(count, line)
# Remove comments
line = comment_re.sub(b'', line)
# Remove leading and trailing whitespace
line = line.strip()
if not line:
continue
m = name_and_email_re.match(line)
if not m:
raise SystemExit(err)
proper_name, proper_email = m.groups()
if len(line) == m.end():
self.changes[(None, proper_email)] = (proper_name, proper_email)
continue
rest = line[m.end():]
m = name_and_email_re.match(rest)
if m:
commit_name, commit_email = m.groups()
if len(rest) != m.end():
raise SystemExit(err)
else:
commit_name, commit_email = rest, None
self.changes[(commit_name, commit_email)] = (proper_name, proper_email)
def translate(self, name, email):
''' Given a name and email, return the expected new name and email from the
mailmap if there is a translation rule for it, otherwise just return
the given name and email.'''
for old, new in self.changes.items():
old_name, old_email = old
new_name, new_email = new
if (old_email is None or email.lower() == old_email.lower()) and (
name == old_name or not old_name):
return (new_name or name, new_email or email)
return (name, email)
class ProgressWriter(object):
def __init__(self):
self._last_progress_update = time.time()
self._last_message = None
def show(self, msg):
self._last_message = msg
now = time.time()
if now - self._last_progress_update > .1:
self._last_progress_update = now
sys.stdout.write("\r{}".format(msg))
sys.stdout.flush()
def finish(self):
self._last_progress_update = 0
if self._last_message:
self.show(self._last_message)
sys.stdout.write("\n")
class _IDs(object):
"""
A class that maintains the 'name domain' of all the 'marks' (short int
id for a blob/commit git object). The reason this mechanism is necessary
is because the text of fast-export may refer to an object using a different
mark than the mark that was assigned to that object using IDS.new(). This
class allows you to translate the fast-export marks (old) to the marks
assigned from IDS.new() (new).
Note that there are two reasons why the marks may differ: (1) The
user manually creates Blob or Commit objects (for insertion into the
stream) (2) We're reading the data from two different repositories
and trying to combine the data (git fast-export will number ids from
1...n, and having two 1's, two 2's, two 3's, causes issues).
"""
def __init__(self):
"""
Init
"""
# The id for the next created blob/commit object
self._next_id = 1
# A map of old-ids to new-ids (1:1 map)
self._translation = {}
# A map of new-ids to every old-id that points to the new-id (1:N map)
self._reverse_translation = {}
def has_renames(self):
"""
Return whether there have been ids remapped to new values
"""
return bool(self._translation)
def new(self):
"""
Should be called whenever a new blob or commit object is created. The
returned value should be used as the id/mark for that object.
"""
rv = self._next_id
self._next_id += 1
return rv
def record_rename(self, old_id, new_id, handle_transitivity = False):
"""
Record that old_id is being renamed to new_id.
"""
if old_id != new_id:
# old_id -> new_id
self._translation[old_id] = new_id
# Transitivity will be needed if new commits are being inserted mid-way
# through a branch.
if handle_transitivity:
# Anything that points to old_id should point to new_id
if old_id in self._reverse_translation:
for id_ in self._reverse_translation[old_id]:
self._translation[id_] = new_id
# Record that new_id is pointed to by old_id
if new_id not in self._reverse_translation:
self._reverse_translation[new_id] = []
self._reverse_translation[new_id].append(old_id)
def translate(self, old_id):
"""
If old_id has been mapped to an alternate id, return the alternate id.
"""
if old_id in self._translation:
return self._translation[old_id]
else:
return old_id
def __str__(self):
"""
Convert IDs to string; used for debugging
"""
rv = "Current count: %d\nTranslation:\n" % self._next_id
for k in sorted(self._translation):
rv += " %d -> %s\n" % (k, self._translation[k])
rv += "Reverse translation:\n"
for k in sorted(self._reverse_translation):
rv += " " + str(k) + " -> " + str(self._reverse_translation[k]) + "\n"
return rv
class _GitElement(object):
"""
The base class for all git elements that we create.
"""
def __init__(self):
# A string that describes what type of Git element this is
self.type = None
# A flag telling us if this Git element has been dumped
# (i.e. printed) or skipped. Typically elements that have been
# dumped or skipped will not be dumped again.
self.dumped = 0
def dump(self, file_):
"""
This version should never be called. Derived classes need to
override! We should note that subclasses should implement this
method such that the output would match the format produced by
fast-export.
"""
raise SystemExit(_("Unimplemented function: %s") % type(self).__name__
+".dump()") # pragma: no cover
def __bytes__(self):
"""
Convert GitElement to bytestring; used for debugging
"""
old_dumped = self.dumped
writeme = io.BytesIO()
self.dump(writeme)
output_lines = writeme.getvalue().splitlines()
writeme.close()
self.dumped = old_dumped
return b"%s:\n %s" % (type(self).__name__.encode(),
b"\n ".join(output_lines))
def skip(self, new_id=None):
"""
Ensures this element will not be written to output
"""
self.dumped = 2
class _GitElementWithId(_GitElement):
"""
The base class for Git elements that have IDs (commits and blobs)
"""
def __init__(self):
_GitElement.__init__(self)
# The mark (short, portable id) for this element
self.id = _IDS.new()
# The previous mark for this element
self.old_id = None
def skip(self, new_id=None):
"""
This element will no longer be automatically written to output. When a
commit gets skipped, it's ID will need to be translated to that of its
parent.
"""
self.dumped = 2
_IDS.record_rename(self.old_id or self.id, new_id)
class Blob(_GitElementWithId):
"""
This class defines our representation of git blob elements (i.e. our
way of representing file contents).
"""
def __init__(self, data, original_id = None):
_GitElementWithId.__init__(self)
# Denote that this is a blob
self.type = 'blob'
# Record original id
self.original_id = original_id
# Stores the blob's data
assert(type(data) == bytes)
self.data = data
def dump(self, file_):
"""
Write this blob element to a file.
"""
self.dumped = 1
HASH_TO_ID[self.original_id] = self.id
ID_TO_HASH[self.id] = self.original_id
file_.write(b'blob\n')
file_.write(b'mark :%d\n' % self.id)
file_.write(b'data %d\n%s' % (len(self.data), self.data))
file_.write(b'\n')
class Reset(_GitElement):
"""
This class defines our representation of git reset elements. A reset
event is the creation (or recreation) of a named branch, optionally
starting from a specific revision).
"""
def __init__(self, ref, from_ref = None):
_GitElement.__init__(self)
# Denote that this is a reset
self.type = 'reset'
# The name of the branch being (re)created
self.ref = ref
# Some reference to the branch/commit we are resetting from
self.from_ref = from_ref
def dump(self, file_):
"""
Write this reset element to a file
"""
self.dumped = 1
file_.write(b'reset %s\n' % self.ref)
if self.from_ref:
if isinstance(self.from_ref, int):
file_.write(b'from :%d\n' % self.from_ref)
else:
file_.write(b'from %s\n' % self.from_ref)
file_.write(b'\n')
class FileChange(_GitElement):
"""
This class defines our representation of file change elements. File change
elements are components within a Commit element.
"""
def __init__(self, type_, filename = None, id_ = None, mode = None):
_GitElement.__init__(self)
# Denote the type of file-change (b'M' for modify, b'D' for delete, etc)
# We could
# assert(type(type_) == bytes)
# here but I don't just due to worries about performance overhead...
self.type = type_
# Record the name of the file being changed
self.filename = filename
# Record the mode (mode describes type of file entry (non-executable,
# executable, or symlink)).
self.mode = mode
# blob_id is the id (mark) of the affected blob
self.blob_id = id_
if type_ == b'DELETEALL':
assert filename is None and id_ is None and mode is None
self.filename = b'' # Just so PathQuoting.enquote doesn't die
else:
assert filename is not None
if type_ == b'M':
assert id_ is not None and mode is not None
elif type_ == b'D':
assert id_ is None and mode is None
elif type_ == b'R': # pragma: no cover (now avoid fast-export renames)
assert mode is None
if id_ is None:
raise SystemExit(_("new name needed for rename of %s") % filename)
self.filename = (self.filename, id_)
self.blob_id = None
def dump(self, file_):
"""
Write this file-change element to a file
"""
skipped_blob = (self.type == b'M' and self.blob_id is None)
if skipped_blob: return
self.dumped = 1
quoted_filename = PathQuoting.enquote(self.filename)
if self.type == b'M' and isinstance(self.blob_id, int):
file_.write(b'M %s :%d %s\n' % (self.mode, self.blob_id, quoted_filename))
elif self.type == b'M':
file_.write(b'M %s %s %s\n' % (self.mode, self.blob_id, quoted_filename))
elif self.type == b'D':
file_.write(b'D %s\n' % quoted_filename)
elif self.type == b'DELETEALL':
file_.write(b'deleteall\n')
else:
raise SystemExit(_("Unhandled filechange type: %s") % self.type) # pragma: no cover
class Commit(_GitElementWithId):
"""
This class defines our representation of commit elements. Commit elements
contain all the information associated with a commit.
"""
def __init__(self, branch,
author_name, author_email, author_date,
committer_name, committer_email, committer_date,
message,
file_changes,
parents,
original_id = None,
encoding = None, # encoding for message; None implies UTF-8
**kwargs):
_GitElementWithId.__init__(self)
self.old_id = self.id
# Denote that this is a commit element
self.type = 'commit'
# Record the affected branch
self.branch = branch
# Record original id
self.original_id = original_id
# Record author's name
self.author_name = author_name
# Record author's email
self.author_email = author_email
# Record date of authoring
self.author_date = author_date
# Record committer's name
self.committer_name = committer_name
# Record committer's email
self.committer_email = committer_email
# Record date the commit was made
self.committer_date = committer_date
# Record commit message and its encoding
self.encoding = encoding
self.message = message
# List of file-changes associated with this commit. Note that file-changes
# are also represented as git elements
self.file_changes = file_changes
self.parents = parents
def dump(self, file_):
"""
Write this commit element to a file.
"""
self.dumped = 1
HASH_TO_ID[self.original_id] = self.id
ID_TO_HASH[self.id] = self.original_id
# Make output to fast-import slightly easier for humans to read if the
# message has no trailing newline of its own; cosmetic, but a nice touch...
extra_newline = b'\n'
if self.message.endswith(b'\n') or not (self.parents or self.file_changes):
extra_newline = b''
if not self.parents:
file_.write(b'reset %s\n' % self.branch)
file_.write((b'commit %s\n'
b'mark :%d\n'
b'author %s <%s> %s\n'
b'committer %s <%s> %s\n'
) % (
self.branch, self.id,
self.author_name, self.author_email, self.author_date,
self.committer_name, self.committer_email, self.committer_date
))
if self.encoding:
file_.write(b'encoding %s\n' % self.encoding)
file_.write(b'data %d\n%s%s' %
(len(self.message), self.message, extra_newline))
for i, parent in enumerate(self.parents):
file_.write(b'from ' if i==0 else b'merge ')
if isinstance(parent, int):
file_.write(b':%d\n' % parent)
else:
file_.write(b'%s\n' % parent)
for change in self.file_changes:
change.dump(file_)
if not self.parents and not self.file_changes:
# Workaround a bug in pre-git-2.22 versions of fast-import with
# the get-mark directive.
file_.write(b'\n')
file_.write(b'\n')
def first_parent(self):
"""
Return first parent commit
"""
if self.parents:
return self.parents[0]
return None
def skip(self, new_id=None):
_SKIPPED_COMMITS.add(self.old_id or self.id)
_GitElementWithId.skip(self, new_id)
class Tag(_GitElementWithId):
"""
This class defines our representation of annotated tag elements.
"""
def __init__(self, ref, from_ref,
tagger_name, tagger_email, tagger_date, tag_msg,
original_id = None):
_GitElementWithId.__init__(self)
self.old_id = self.id
# Denote that this is a tag element
self.type = 'tag'
# Store the name of the tag
self.ref = ref
# Store the entity being tagged (this should be a commit)
self.from_ref = from_ref
# Record original id
self.original_id = original_id
# Store the name of the tagger
self.tagger_name = tagger_name
# Store the email of the tagger
self.tagger_email = tagger_email
# Store the date
self.tagger_date = tagger_date
# Store the tag message
self.message = tag_msg
def dump(self, file_):
"""
Write this tag element to a file
"""
self.dumped = 1
HASH_TO_ID[self.original_id] = self.id
ID_TO_HASH[self.id] = self.original_id
file_.write(b'tag %s\n' % self.ref)
if (write_marks and self.id):
file_.write(b'mark :%d\n' % self.id)
markfmt = b'from :%d\n' if isinstance(self.from_ref, int) else b'from %s\n'
file_.write(markfmt % self.from_ref)
if self.tagger_name:
file_.write(b'tagger %s <%s> ' % (self.tagger_name, self.tagger_email))
file_.write(self.tagger_date)
file_.write(b'\n')
file_.write(b'data %d\n%s' % (len(self.message), self.message))
file_.write(b'\n')
class Progress(_GitElement):
"""
This class defines our representation of progress elements. The progress
element only contains a progress message, which is printed by fast-import
when it processes the progress output.
"""
def __init__(self, message):
_GitElement.__init__(self)
# Denote that this is a progress element
self.type = 'progress'
# Store the progress message
self.message = message
def dump(self, file_):
"""
Write this progress element to a file
"""
self.dumped = 1
file_.write(b'progress %s\n' % self.message)
file_.write(b'\n')
class Checkpoint(_GitElement):
"""
This class defines our representation of checkpoint elements. These
elements represent events which force fast-import to close the current
packfile, start a new one, and to save out all current branch refs, tags
and marks.
"""
def __init__(self):
_GitElement.__init__(self)
# Denote that this is a checkpoint element
self.type = 'checkpoint'
def dump(self, file_):
"""
Write this checkpoint element to a file
"""
self.dumped = 1
file_.write(b'checkpoint\n')
file_.write(b'\n')
class LiteralCommand(_GitElement):
"""
This class defines our representation of commands. The literal command
includes only a single line, and is not processed in any special way.
"""
def __init__(self, line):
_GitElement.__init__(self)
# Denote that this is a literal element
self.type = 'literal'
# Store the command
self.line = line
def dump(self, file_):
"""
Write this progress element to a file
"""
self.dumped = 1
file_.write(self.line)
class Alias(_GitElement):
"""
This class defines our representation of fast-import alias elements. An
alias element is the setting of one mark to the same sha1sum as another,
usually because the newer mark corresponded to a pruned commit.
"""
def __init__(self, ref, to_ref):
_GitElement.__init__(self)
# Denote that this is a reset
self.type = 'alias'
self.ref = ref
self.to_ref = to_ref
def dump(self, file_):
"""
Write this reset element to a file
"""
self.dumped = 1
file_.write(b'alias\nmark :%d\nto :%d\n\n' % (self.ref, self.to_ref))
class FastExportParser(object):
"""
A class for parsing and handling the output from fast-export. This
class allows the user to register callbacks when various types of
data are encountered in the fast-export output. The basic idea is that,
FastExportParser takes fast-export output, creates the various objects
as it encounters them, the user gets to use/modify these objects via
callbacks, and finally FastExportParser outputs the modified objects
in fast-import format (presumably so they can be used to create a new
repo).
"""
def __init__(self,
tag_callback = None, commit_callback = None,
blob_callback = None, progress_callback = None,
reset_callback = None, checkpoint_callback = None,
done_callback = None):
# Members below simply store callback functions for the various git
# elements
self._tag_callback = tag_callback
self._blob_callback = blob_callback
self._reset_callback = reset_callback
self._commit_callback = commit_callback
self._progress_callback = progress_callback
self._checkpoint_callback = checkpoint_callback
self._done_callback = done_callback
# Keep track of which refs appear from the export, and which make it to
# the import (pruning of empty commits, renaming of refs, and creating
# new manual objects and inserting them can cause these to differ).
self._exported_refs = set()
self._imported_refs = set()
# A list of the branches we've seen, plus the last known commit they
# pointed to. An entry in latest_*commit will be deleted if we get a
# reset for that branch. These are used because of fast-import's weird
# decision to allow having an implicit parent via naming the branch
# instead of requiring branches to be specified via 'from' directives.
self._latest_commit = {}
self._latest_orig_commit = {}
# A handle to the input source for the fast-export data
self._input = None
# A handle to the output file for the output we generate (we call dump
# on many of the git elements we create).
self._output = None
# Stores the contents of the current line of input being parsed
self._currentline = ''
# Compile some regexes and cache those
self._mark_re = re.compile(br'mark :(\d+)\n$')
self._parent_regexes = {}
parent_regex_rules = (br' :(\d+)\n$', br' ([0-9a-f]{40})\n')
for parent_refname in (b'from', b'merge'):
ans = [re.compile(parent_refname+x) for x in parent_regex_rules]
self._parent_regexes[parent_refname] = ans
self._quoted_string_re = re.compile(br'"(?:[^"\\]|\\.)*"')
self._refline_regexes = {}
for refline_name in (b'reset', b'commit', b'tag', b'progress'):
self._refline_regexes[refline_name] = re.compile(refline_name+b' (.*)\n$')
self._user_regexes = {}
for user in (b'author', b'committer', b'tagger'):
self._user_regexes[user] = re.compile(user + b' (.*?) <(.*?)> (.*)\n$')
def _advance_currentline(self):
"""
Grab the next line of input
"""
self._currentline = self._input.readline()
def _parse_optional_mark(self):
"""
If the current line contains a mark, parse it and advance to the
next line; return None otherwise
"""
mark = None
matches = self._mark_re.match(self._currentline)
if matches:
mark = int(matches.group(1))
self._advance_currentline()
return mark
def _parse_optional_parent_ref(self, refname):
"""
If the current line contains a reference to a parent commit, then
parse it and advance the current line; otherwise return None. Note
that the name of the reference ('from', 'merge') must match the
refname arg.
"""
orig_baseref, baseref = None, None
rule, altrule = self._parent_regexes[refname]
matches = rule.match(self._currentline)
if matches:
orig_baseref = int(matches.group(1))
# We translate the parent commit mark to what it needs to be in
# our mark namespace
baseref = _IDS.translate(orig_baseref)
self._advance_currentline()
else:
matches = altrule.match(self._currentline)
if matches:
orig_baseref = matches.group(1)
baseref = orig_baseref
self._advance_currentline()
return orig_baseref, baseref
def _parse_optional_filechange(self):
"""
If the current line contains a file-change object, then parse it
and advance the current line; otherwise return None. We only care
about file changes of type b'M' and b'D' (these are the only types
of file-changes that fast-export will provide).
"""
filechange = None
changetype = self._currentline[0:1]
if changetype == b'M':
(changetype, mode, idnum, path) = self._currentline.split(None, 3)
if idnum[0:1] == b':':
idnum = idnum[1:]
path = path.rstrip(b'\n')
# We translate the idnum to our id system
if len(idnum) != 40:
idnum = _IDS.translate( int(idnum) )
if idnum is not None:
if path.startswith(b'"'):
path = PathQuoting.dequote(path)
filechange = FileChange(b'M', path, idnum, mode)
else:
filechange = b'skipped'
self._advance_currentline()
elif changetype == b'D':
(changetype, path) = self._currentline.split(None, 1)
path = path.rstrip(b'\n')
if path.startswith(b'"'):
path = PathQuoting.dequote(path)
filechange = FileChange(b'D', path)
self._advance_currentline()
elif changetype == b'R': # pragma: no cover (now avoid fast-export renames)
rest = self._currentline[2:-1]
if rest.startswith(b'"'):
m = self._quoted_string_re.match(rest)
if not m:
raise SystemExit(_("Couldn't parse rename source"))
orig = PathQuoting.dequote(m.group(0))
new = rest[m.end()+1:]
else:
orig, new = rest.split(b' ', 1)
if new.startswith(b'"'):
new = PathQuoting.dequote(new)
filechange = FileChange(b'R', orig, new)
self._advance_currentline()
return filechange
def _parse_original_id(self):
original_id = self._currentline[len(b'original-oid '):].rstrip()
self._advance_currentline()
return original_id
def _parse_encoding(self):
encoding = self._currentline[len(b'encoding '):].rstrip()
self._advance_currentline()
return encoding
def _parse_ref_line(self, refname):
"""
Parses string data (often a branch name) from current-line. The name of
the string data must match the refname arg. The program will crash if
current-line does not match, so current-line will always be advanced if
this method returns.
"""
matches = self._refline_regexes[refname].match(self._currentline)
if not matches:
raise SystemExit(_("Malformed %(refname)s line: '%(line)s'") %
({'refname': refname, 'line':self._currentline})
) # pragma: no cover
ref = matches.group(1)
self._advance_currentline()
return ref
def _parse_user(self, usertype):
"""
Get user name, email, datestamp from current-line. Current-line will
be advanced.
"""
user_regex = self._user_regexes[usertype]
(name, email, when) = user_regex.match(self._currentline).groups()
self._advance_currentline()
return (name, email, when)
def _parse_data(self):
"""
Reads data from _input. Current-line will be advanced until it is beyond
the data.
"""
fields = self._currentline.split()
assert fields[0] == b'data'
size = int(fields[1])
data = self._input.read(size)
self._advance_currentline()
if self._currentline == b'\n':
self._advance_currentline()
return data
def _parse_blob(self):
"""
Parse input data into a Blob object. Once the Blob has been created, it
will be handed off to the appropriate callbacks. Current-line will be
advanced until it is beyond this blob's data. The Blob will be dumped
to _output once everything else is done (unless it has been skipped by
the callback).
"""
# Parse the Blob
self._advance_currentline()
id_ = self._parse_optional_mark()
original_id = None
if self._currentline.startswith(b'original-oid'):
original_id = self._parse_original_id();
data = self._parse_data()
if self._currentline == b'\n':
self._advance_currentline()
# Create the blob
blob = Blob(data, original_id)
# If fast-export text had a mark for this blob, need to make sure this
# mark translates to the blob's true id.
if id_:
blob.old_id = id_
_IDS.record_rename(id_, blob.id)
# Call any user callback to allow them to use/modify the blob
if self._blob_callback:
self._blob_callback(blob)
# Now print the resulting blob
if not blob.dumped:
blob.dump(self._output)
def _parse_reset(self):
"""
Parse input data into a Reset object. Once the Reset has been created,
it will be handed off to the appropriate callbacks. Current-line will
be advanced until it is beyond the reset data. The Reset will be dumped
to _output once everything else is done (unless it has been skipped by
the callback).
"""
# Parse the Reset
ref = self._parse_ref_line(b'reset')
self._exported_refs.add(ref)
ignoreme, from_ref = self._parse_optional_parent_ref(b'from')
if self._currentline == b'\n':
self._advance_currentline()
# fast-export likes to print extraneous resets that serve no purpose.
# While we could continue processing such resets, that is a waste of
# resources. Also, we want to avoid recording that this ref was
# seen in such cases, since this ref could be rewritten to nothing.
if not from_ref:
self._latest_commit.pop(ref, None)
self._latest_orig_commit.pop(ref, None)
return
# Create the reset
reset = Reset(ref, from_ref)
# Call any user callback to allow them to modify the reset
if self._reset_callback:
self._reset_callback(reset)
# Update metadata
self._latest_commit[reset.ref] = reset.from_ref
self._latest_orig_commit[reset.ref] = reset.from_ref
# Now print the resulting reset
if not reset.dumped:
self._imported_refs.add(reset.ref)
reset.dump(self._output)
def _parse_commit(self):
"""
Parse input data into a Commit object. Once the Commit has been created,
it will be handed off to the appropriate callbacks. Current-line will
be advanced until it is beyond the commit data. The Commit will be dumped
to _output once everything else is done (unless it has been skipped by
the callback OR the callback has removed all file-changes from the commit).
"""
# Parse the Commit. This may look involved, but it's pretty simple; it only
# looks bad because a commit object contains many pieces of data.
branch = self._parse_ref_line(b'commit')
self._exported_refs.add(branch)
id_ = self._parse_optional_mark()
original_id = None
if self._currentline.startswith(b'original-oid'):
original_id = self._parse_original_id();
author_name = None
author_email = None
if self._currentline.startswith(b'author'):
(author_name, author_email, author_date) = self._parse_user(b'author')
(committer_name, committer_email, committer_date) = \
self._parse_user(b'committer')
if not author_name and not author_email:
(author_name, author_email, author_date) = \
(committer_name, committer_email, committer_date)
encoding = None
if self._currentline.startswith(b'encoding '):
encoding = self._parse_encoding()
commit_msg = self._parse_data()
pinfo = [self._parse_optional_parent_ref(b'from')]
# Due to empty pruning, we can have real 'from' and 'merge' lines that
# due to commit rewriting map to a parent of None. We need to record
# 'from' if its non-None, and we need to parse all 'merge' lines.
while self._currentline.startswith(b'merge '):
pinfo.append(self._parse_optional_parent_ref(b'merge'))
orig_parents, parents = [list(tmp) for tmp in zip(*pinfo)]
# No parents is oddly represented as [None] instead of [], due to the
# special 'from' handling. Convert it here to a more canonical form.
if parents == [None]:
parents = []
if orig_parents == [None]:
orig_parents = []
# fast-import format is kinda stupid in that it allows implicit parents
# based on the branch name instead of requiring them to be specified by
# 'from' directives. The only way to get no parent is by using a reset
# directive first, which clears the latest_commit_for_this_branch tracking.
if not orig_parents and self._latest_commit.get(branch):
parents = [self._latest_commit[branch]]
if not orig_parents and self._latest_orig_commit.get(branch):
orig_parents = [self._latest_orig_commit[branch]]
# Get the list of file changes
file_changes = []
file_change = self._parse_optional_filechange()
had_file_changes = file_change is not None
while file_change:
if not (type(file_change) == bytes and file_change == b'skipped'):
file_changes.append(file_change)
file_change = self._parse_optional_filechange()
if self._currentline == b'\n':
self._advance_currentline()
# Okay, now we can finally create the Commit object
commit = Commit(branch,
author_name, author_email, author_date,
committer_name, committer_email, committer_date,
commit_msg, file_changes, parents, original_id, encoding)
# If fast-export text had a mark for this commit, need to make sure this
# mark translates to the commit's true id.
if id_:
commit.old_id = id_
_IDS.record_rename(id_, commit.id)
# Call any user callback to allow them to modify the commit
aux_info = {'orig_parents': orig_parents,
'had_file_changes': had_file_changes}
if self._commit_callback:
self._commit_callback(commit, aux_info)
# Now print the resulting commit, or if prunable skip it
self._latest_orig_commit[branch] = commit.id
if not (commit.old_id or commit.id) in _SKIPPED_COMMITS:
self._latest_commit[branch] = commit.id
if not commit.dumped:
self._imported_refs.add(commit.branch)
commit.dump(self._output)
def _parse_tag(self):
"""
Parse input data into a Tag object. Once the Tag has been created,
it will be handed off to the appropriate callbacks. Current-line will
be advanced until it is beyond the tag data. The Tag will be dumped
to _output once everything else is done (unless it has been skipped by
the callback).
"""
# Parse the Tag
tag = self._parse_ref_line(b'tag')
self._exported_refs.add(b'refs/tags/'+tag)
id_ = self._parse_optional_mark()
ignoreme, from_ref = self._parse_optional_parent_ref(b'from')
original_id = None
if self._currentline.startswith(b'original-oid'):
original_id = self._parse_original_id();
tagger_name, tagger_email, tagger_date = None, None, None
if self._currentline.startswith(b'tagger'):
(tagger_name, tagger_email, tagger_date) = self._parse_user(b'tagger')
tag_msg = self._parse_data()
if self._currentline == b'\n':
self._advance_currentline()
# Create the tag
tag = Tag(tag, from_ref,
tagger_name, tagger_email, tagger_date, tag_msg,
original_id)
# If fast-export text had a mark for this tag, need to make sure this
# mark translates to the tag's true id.
if id_:
tag.old_id = id_
_IDS.record_rename(id_, tag.id)
# Call any user callback to allow them to modify the tag
if self._tag_callback:
self._tag_callback(tag)
# The tag might not point at anything that still exists (self.from_ref
# will be None if the commit it pointed to and all its ancestors were
# pruned due to being empty)
if tag.from_ref:
# Print out this tag's information
if not tag.dumped:
self._imported_refs.add(b'refs/tags/'+tag.ref)
tag.dump(self._output)
else:
tag.skip()
def _parse_progress(self):
"""
Parse input data into a Progress object. Once the Progress has
been created, it will be handed off to the appropriate
callbacks. Current-line will be advanced until it is beyond the
progress data. The Progress will be dumped to _output once
everything else is done (unless it has been skipped by the callback).
"""
# Parse the Progress
message = self._parse_ref_line(b'progress')
if self._currentline == b'\n':
self._advance_currentline()
# Create the progress message
progress = Progress(message)
# Call any user callback to allow them to modify the progress messsage
if self._progress_callback:
self._progress_callback(progress)
# NOTE: By default, we do NOT print the progress message; git
# fast-import would write it to fast_import_pipes which could mess with
# our parsing of output from the 'ls' and 'get-mark' directives we send
# to fast-import. If users want these messages, they need to process
# and handle them in the appropriate callback above.
def _parse_checkpoint(self):
"""
Parse input data into a Checkpoint object. Once the Checkpoint has
been created, it will be handed off to the appropriate
callbacks. Current-line will be advanced until it is beyond the
checkpoint data. The Checkpoint will be dumped to _output once
everything else is done (unless it has been skipped by the callback).
"""
# Parse the Checkpoint
self._advance_currentline()
if self._currentline == b'\n':
self._advance_currentline()
# Create the checkpoint
checkpoint = Checkpoint()
# Call any user callback to allow them to drop the checkpoint
if self._checkpoint_callback:
self._checkpoint_callback(checkpoint)
# NOTE: By default, we do NOT print the checkpoint message; although it
# we would only realistically get them with --stdin, the fact that we
# are filtering makes me think the checkpointing is less likely to be
# reasonable. In fact, I don't think it's necessary in general. If
# users do want it, they should process it in the checkpoint_callback.
def _parse_literal_command(self):
"""
Parse literal command. Then just dump the line as is.
"""
# Create the literal command object
command = LiteralCommand(self._currentline)
self._advance_currentline()
# Now print the resulting literal command
if not command.dumped:
command.dump(self._output)
def insert(self, obj):
assert not obj.dumped
obj.dump(self._output)
if type(obj) == Commit:
self._imported_refs.add(obj.branch)
elif type(obj) in (Reset, Tag):
self._imported_refs.add(obj.ref)
def run(self, input, output):
"""
This method filters fast export output.
"""
# Set input. If no args provided, use stdin.
self._input = input
self._output = output
# Run over the input and do the filtering
self._advance_currentline()
while self._currentline:
if self._currentline.startswith(b'blob'):
self._parse_blob()
elif self._currentline.startswith(b'reset'):
self._parse_reset()
elif self._currentline.startswith(b'commit'):
self._parse_commit()
elif self._currentline.startswith(b'tag'):
self._parse_tag()
elif self._currentline.startswith(b'progress'):
self._parse_progress()
elif self._currentline.startswith(b'checkpoint'):
self._parse_checkpoint()
elif self._currentline.startswith(b'feature'):
self._parse_literal_command()
elif self._currentline.startswith(b'option'):
self._parse_literal_command()
elif self._currentline.startswith(b'done'):
if self._done_callback:
self._done_callback()
self._parse_literal_command()
# Prevent confusion from others writing additional stuff that'll just
# be ignored
self._output.close()
elif self._currentline.startswith(b'#'):
self._parse_literal_command()
elif self._currentline.startswith(b'get-mark') or \
self._currentline.startswith(b'cat-blob') or \
self._currentline.startswith(b'ls'):
raise SystemExit(_("Unsupported command: '%s'") % self._currentline)
else:
raise SystemExit(_("Could not parse line: '%s'") % self._currentline)
def get_exported_and_imported_refs(self):
return self._exported_refs, self._imported_refs
def record_id_rename(old_id, new_id):
"""
Register a new translation
"""
handle_transitivity = True
_IDS.record_rename(old_id, new_id, handle_transitivity)
# Internal globals
_IDS = _IDs()
_SKIPPED_COMMITS = set()
HASH_TO_ID = {}
ID_TO_HASH = {}
class SubprocessWrapper(object):
@staticmethod
def decodify(args):
if type(args) == str:
return args
else:
assert type(args) == list
return [decode(x) if type(x)==bytes else x for x in args]
@staticmethod
def call(*args, **kwargs):
if 'cwd' in kwargs:
kwargs['cwd'] = decode(kwargs['cwd'])
return subprocess.call(SubprocessWrapper.decodify(*args), **kwargs)
@staticmethod
def check_output(*args, **kwargs):
if 'cwd' in kwargs:
kwargs['cwd'] = decode(kwargs['cwd'])
return subprocess.check_output(SubprocessWrapper.decodify(*args), **kwargs)
@staticmethod
def check_call(*args, **kwargs): # pragma: no cover # used by filter-lamely
if 'cwd' in kwargs:
kwargs['cwd'] = decode(kwargs['cwd'])
return subprocess.check_call(SubprocessWrapper.decodify(*args), **kwargs)
@staticmethod
def Popen(*args, **kwargs):
if 'cwd' in kwargs:
kwargs['cwd'] = decode(kwargs['cwd'])
return subprocess.Popen(SubprocessWrapper.decodify(*args), **kwargs)
subproc = subprocess
if platform.system() == 'Windows' or 'PRETEND_UNICODE_ARGS' in os.environ:
subproc = SubprocessWrapper
class GitUtils(object):
@staticmethod
def get_commit_count(repo, *args):
"""
Return the number of commits that have been made on repo.
"""
if not args:
args = ['--all']
if len(args) == 1 and isinstance(args[0], list):
args = args[0]
p = subproc.Popen(["git", "rev-list", "--count"] + args,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
cwd=repo)
if p.wait() != 0:
raise SystemExit(_("%s does not appear to be a valid git repository")
% decode(repo))
return int(p.stdout.read())
@staticmethod
def get_total_objects(repo):
"""
Return the number of objects (both packed and unpacked)
"""
p1 = subproc.Popen(["git", "count-objects", "-v"],
stdout=subprocess.PIPE, cwd=repo)
lines = p1.stdout.read().splitlines()
# Return unpacked objects + packed-objects
return int(lines[0].split()[1]) + int(lines[2].split()[1])
@staticmethod
def is_repository_bare(repo_working_dir):
out = subproc.check_output('git rev-parse --is-bare-repository'.split(),
cwd=repo_working_dir)
return (out.strip() == b'true')
@staticmethod
def determine_git_dir(repo_working_dir):
d = subproc.check_output('git rev-parse --git-dir'.split(),
cwd=repo_working_dir).strip()
if repo_working_dir==b'.' or d.startswith(b'/'):
return d
return os.path.join(repo_working_dir, d)
@staticmethod
def get_refs(repo_working_dir):
try:
output = subproc.check_output('git show-ref'.split(),
cwd=repo_working_dir)
except subprocess.CalledProcessError as e:
# If error code is 1, there just aren't any refs; i.e. new repo.
# If error code is other than 1, some other error (e.g. not a git repo)
if e.returncode != 1:
raise SystemExit('fatal: {}'.format(e))
output = ''
return dict(reversed(x.split()) for x in output.splitlines())
@staticmethod
def get_blob_sizes(quiet = False):
blob_size_progress = ProgressWriter()
num_blobs = 0
processed_blobs_msg = _("Processed %d blob sizes")
# Get sizes of blobs by sha1
cmd = '--batch-check=%(objectname) %(objecttype) ' + \
'%(objectsize) %(objectsize:disk)'
cf = subproc.Popen(['git', 'cat-file', '--batch-all-objects', cmd],
bufsize = -1,
stdout = subprocess.PIPE)
unpacked_size = {}
packed_size = {}
for line in cf.stdout:
sha, objtype, objsize, objdisksize = line.split()
objsize, objdisksize = int(objsize), int(objdisksize)
if objtype == b'blob':
unpacked_size[sha] = objsize
packed_size[sha] = objdisksize
num_blobs += 1
if not quiet:
blob_size_progress.show(processed_blobs_msg % num_blobs)
cf.wait()
if not quiet:
blob_size_progress.finish()
return unpacked_size, packed_size
@staticmethod
def get_file_changes(repo, parent_hash, commit_hash):
"""
Return a FileChanges list with the differences between parent_hash
and commit_hash
"""
file_changes = []
cmd = ["git", "diff-tree", "-r", parent_hash, commit_hash]
output = subproc.check_output(cmd, cwd=repo)
for line in output.splitlines():
fileinfo, path = line.split(b'\t', 1)
if path.startswith(b'"'):
path = PathQuoting.dequote(path)
oldmode, mode, oldhash, newhash, changetype = fileinfo.split()
if changetype == b'D':
file_changes.append(FileChange(b'D', path))
elif changetype in (b'A', b'M', b'T'):
identifier = HASH_TO_ID.get(newhash, newhash)
file_changes.append(FileChange(b'M', path, identifier, mode))
else: # pragma: no cover
raise SystemExit("Unknown change type for line {}".format(line))
return file_changes
@staticmethod
def print_my_version():
with open(__file__, 'br') as f:
contents = f.read()
# If people replaced @@LOCALEDIR@@ string to point at their local
# directory, undo it so we can get original source version.
contents = re.sub(br'\A#\!.*',
br'#!/usr/bin/env python3', contents)
contents = re.sub(br'(\("GIT_TEXTDOMAINDIR"\) or ").*"',
br'\1@@LOCALEDIR@@"', contents)
cmd = 'git hash-object --stdin'.split()
version = subproc.check_output(cmd, input=contents).strip()
print(decode(version[0:12]))
class FilteringOptions(object):
default_replace_text = b'***REMOVED***'
class AppendFilter(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
user_path = values
suffix = option_string[len('--path-'):] or 'match'
if suffix.startswith('rename'):
mod_type = 'rename'
match_type = option_string[len('--path-rename-'):] or 'match'
values = values.split(b':')
if len(values) != 2:
raise SystemExit(_("Error: --path-rename expects one colon in its"
" argument: <old_name:new_name>."))
if values[0] and values[1] and not (
values[0].endswith(b'/') == values[1].endswith(b'/')):
raise SystemExit(_("Error: With --path-rename, if OLD_NAME and "
"NEW_NAME are both non-empty and either ends "
"with a slash then both must."))
if any(v.startswith(b'/') for v in values):
raise SystemExit(_("Error: Pathnames cannot begin with a '/'"))
components = values[0].split(b'/') + values[1].split(b'/')
else:
mod_type = 'filter'
match_type = suffix
components = values.split(b'/')
if values.startswith(b'/'):
raise SystemExit(_("Error: Pathnames cannot begin with a '/'"))
for illegal_path in [b'.', b'..']:
if illegal_path in components:
raise SystemExit(_("Error: Invalid path component '%s' found in '%s'")
% (decode(illegal_path), decode(user_path)))
if match_type == 'regex':
values = re.compile(values)
items = getattr(namespace, self.dest, []) or []
items.append((mod_type, match_type, values))
if (match_type, mod_type) == ('glob', 'filter'):
if not values.endswith(b'*'):
extension = b'*' if values.endswith(b'/') else b'/*'
items.append((mod_type, match_type, values+extension))
setattr(namespace, self.dest, items)
class HelperFilter(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
af = FilteringOptions.AppendFilter(dest='path_changes',
option_strings=None)
dirname = values if values[-1:] == b'/' else values+b'/'
if option_string == '--subdirectory-filter':
af(parser, namespace, dirname, '--path-match')
af(parser, namespace, dirname+b':', '--path-rename')
elif option_string == '--to-subdirectory-filter':
af(parser, namespace, b':'+dirname, '--path-rename')
else:
raise SystemExit(_("Error: HelperFilter given invalid option_string: %s")
% option_string) # pragma: no cover
class FileWithPathsFilter(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
if not namespace.path_changes:
namespace.path_changes = []
namespace.path_changes += FilteringOptions.get_paths_from_file(values)
@staticmethod
def create_arg_parser():
# Include usage in the summary, so we can put the description first
summary = _('''Rewrite (or analyze) repository history
git-filter-repo destructively rewrites history (unless --analyze or
--dry-run are given) according to specified rules. It refuses to do any
rewriting unless either run from a clean fresh clone, or --force was
given.
Basic Usage:
git-filter-repo --analyze
git-filter-repo [FILTER/RENAME/CONTROL OPTIONS]
See EXAMPLES section for details.
''').rstrip()
# Provide a long helpful examples section
example_text = _('''CALLBACKS
All callback functions are of the same general format. For a command line
argument like
--foo-callback 'BODY'
the following code will be compiled and called:
def foo_callback(foo):
BODY
Thus, to replace 'Jon' with 'John' in author/committer/tagger names:
git filter-repo --name-callback 'return name.replace(b"Jon", b"John")'
To remove all 'Tested-by' tags in commit (or tag) messages:
git filter-repo --message-callback 'return re.sub(br"\\nTested-by:.*", "", message)'
To remove all .DS_Store files:
git filter-repo --filename-callback 'return None if os.path.basename(filename) == b".DS_Store" else filename'
Note that if BODY resolves to a filename, then the contents of that file
will be used as the BODY in the callback function.
For more detailed examples and explanations AND caveats, see
https://htmlpreview.github.io/?https://github.com/newren/git-filter-repo/blob/docs/html/git-filter-repo.html#CALLBACKS
EXAMPLES
To get a bunch of reports mentioning renames that have occurred in
your repo and listing sizes of objects aggregated by any of path,
directory, extension, or blob-id:
git filter-repo --analyze
(These reports can help you choose how to filter your repo; it can
be useful to re-run this command after filtering to regenerate the
report and verify the changes look correct.)
To extract the history that touched just 'guides' and 'tools/releases':
git filter-repo --path guides/ --path tools/releases
To remove foo.zip and bar/baz/zips from every revision in history:
git filter-repo --path foo.zip --path bar/baz/zips/ --invert-paths
To replace the text 'password' with 'p455w0rd':
git filter-repo --replace-text <(echo "password==>p455w0rd")
To use the current version of the .mailmap file to update authors,
committers, and taggers throughout history and make it permanent:
git filter-repo --use-mailmap
To extract the history of 'src/', rename all files to have a new leading
directory 'my-module' (e.g. src/foo.java -> my-module/src/foo.java), and
add a 'my-module-' prefix to all tags:
git filter-repo --path src/ --to-subdirectory-filter my-module --tag-rename '':'my-module-'
For more detailed examples and explanations, see
https://htmlpreview.github.io/?https://github.com/newren/git-filter-repo/blob/docs/html/git-filter-repo.html#EXAMPLES''')
# Create the basic parser
parser = argparse.ArgumentParser(description=summary,
usage = argparse.SUPPRESS,
add_help = False,
epilog = example_text,
formatter_class=argparse.RawDescriptionHelpFormatter)
analyze = parser.add_argument_group(title=_("Analysis"))
analyze.add_argument('--analyze', action='store_true',
help=_("Analyze repository history and create a report that may be "
"useful in determining what to filter in a subsequent run. "
"Will not modify your repo."))
analyze.add_argument('--report-dir',
metavar='DIR_OR_FILE',
type=os.fsencode,
dest='report_dir',
help=_("Directory to write report, defaults to GIT_DIR/filter_repo/analysis,"
"refuses to run if exists, --force delete existing dir first."))
path = parser.add_argument_group(title=_("Filtering based on paths "
"(see also --filename-callback)"),
description=textwrap.dedent(_("""
These options specify the paths to select. Note that much like git
itself, renames are NOT followed so you may need to specify multiple
paths, e.g. `--path olddir/ --path newdir/`
"""[1:])))
path.add_argument('--invert-paths', action='store_false', dest='inclusive',
help=_("Invert the selection of files from the specified "
"--path-{match,glob,regex} options below, i.e. only select "
"files matching none of those options."))
path.add_argument('--path-match', '--path', metavar='DIR_OR_FILE',
type=os.fsencode,
action=FilteringOptions.AppendFilter, dest='path_changes',
help=_("Exact paths (files or directories) to include in filtered "
"history. Multiple --path options can be specified to get "
"a union of paths."))
path.add_argument('--path-glob', metavar='GLOB', type=os.fsencode,
action=FilteringOptions.AppendFilter, dest='path_changes',
help=_("Glob of paths to include in filtered history. Multiple "
"--path-glob options can be specified to get a union of "
"paths."))
path.add_argument('--path-regex', metavar='REGEX', type=os.fsencode,
action=FilteringOptions.AppendFilter, dest='path_changes',
help=_("Regex of paths to include in filtered history. Multiple "
"--path-regex options can be specified to get a union of "
"paths"))
path.add_argument('--use-base-name', action='store_true',
help=_("Match on file base name instead of full path from the top "
"of the repo. Incompatible with --path-rename, and "
"incompatible with matching against directory names."))
rename = parser.add_argument_group(title=_("Renaming based on paths "
"(see also --filename-callback)"))
rename.add_argument('--path-rename', '--path-rename-match',
metavar='OLD_NAME:NEW_NAME', dest='path_changes', type=os.fsencode,
action=FilteringOptions.AppendFilter,
help=_("Path to rename; if filename or directory matches OLD_NAME "
"rename to NEW_NAME. Multiple --path-rename options can be "
"specified. NOTE: If you combine filtering options with "
"renaming ones, do not rely on a rename argument to select "
"paths; you also need a filter to select them."))
helpers = parser.add_argument_group(title=_("Path shortcuts"))
helpers.add_argument('--paths-from-file', metavar='FILENAME',
type=os.fsencode,
action=FilteringOptions.FileWithPathsFilter, dest='path_changes',
help=_("Specify several path filtering and renaming directives, one "
"per line. Lines with '==>' in them specify path renames, "
"and lines can begin with 'literal:' (the default), 'glob:', "
"or 'regex:' to specify different matching styles. Blank "
"lines and lines starting with a '#' are ignored."))
helpers.add_argument('--subdirectory-filter', metavar='DIRECTORY',
action=FilteringOptions.HelperFilter, type=os.fsencode,
help=_("Only look at history that touches the given subdirectory "
"and treat that directory as the project root. Equivalent "
"to using '--path DIRECTORY/ --path-rename DIRECTORY/:'"))
helpers.add_argument('--to-subdirectory-filter', metavar='DIRECTORY',
action=FilteringOptions.HelperFilter, type=os.fsencode,
help=_("Treat the project root as instead being under DIRECTORY. "
"Equivalent to using '--path-rename :DIRECTORY/'"))
contents = parser.add_argument_group(title=_("Content editing filters "
"(see also --blob-callback)"))
contents.add_argument('--replace-text', metavar='EXPRESSIONS_FILE',
help=_("A file with expressions that, if found, will be replaced. "
"By default, each expression is treated as literal text, "
"but 'regex:' and 'glob:' prefixes are supported. You can "
"end the line with '==>' and some replacement text to "
"choose a replacement choice other than the default of '{}'."
.format(decode(FilteringOptions.default_replace_text))))
contents.add_argument('--strip-blobs-bigger-than', metavar='SIZE',
dest='max_blob_size', default=0,
help=_("Strip blobs (files) bigger than specified size (e.g. '5M', "
"'2G', etc)"))
contents.add_argument('--strip-blobs-with-ids', metavar='BLOB-ID-FILENAME',
help=_("Read git object ids from each line of the given file, and "
"strip all of them from history"))
refrename = parser.add_argument_group(title=_("Renaming of refs "
"(see also --refname-callback)"))
refrename.add_argument('--tag-rename', metavar='OLD:NEW', type=os.fsencode,
help=_("Rename tags starting with OLD to start with NEW. For "
"example, --tag-rename foo:bar will rename tag foo-1.2.3 "
"to bar-1.2.3; either OLD or NEW can be empty."))
messages = parser.add_argument_group(title=_("Filtering of commit messages "
"(see also --message-callback)"))
messages.add_argument('--replace-message', metavar='EXPRESSIONS_FILE',
help=_("A file with expressions that, if found in commit messages, "
"will be replaced. This file uses the same syntax as "
"--replace-text."))
messages.add_argument('--preserve-commit-hashes', action='store_true',
help=_("By default, since commits are rewritten and thus gain new "
"hashes, references to old commit hashes in commit messages "
"are replaced with new commit hashes (abbreviated to the same "
"length as the old reference). Use this flag to turn off "
"updating commit hashes in commit messages."))
messages.add_argument('--preserve-commit-encoding', action='store_true',
help=_("Do not reencode commit messages into UTF-8. By default, if "
"the commit object specifies an encoding for the commit "
"message, the message is re-encoded into UTF-8."))
people = parser.add_argument_group(title=_("Filtering of names & emails "
"(see also --name-callback "
"and --email-callback)"))
people.add_argument('--mailmap', dest='mailmap', metavar='FILENAME',
type=os.fsencode,
help=_("Use specified mailmap file (see git-shortlog(1) for "
"details on the format) when rewriting author, committer, "
"and tagger names and emails. If the specified file is "
"part of git history, historical versions of the file will "
"be ignored; only the current contents are consulted."))
people.add_argument('--use-mailmap', dest='mailmap',
action='store_const', const=b'.mailmap',
help=_("Same as: '--mailmap .mailmap' "))
parents = parser.add_argument_group(title=_("Parent rewriting"))
parents.add_argument('--replace-refs', default=None,
choices=['delete-no-add', 'delete-and-add',
'update-no-add', 'update-or-add',
'update-and-add'],
help=_("Replace refs (see git-replace(1)) are used to rewrite "
"parents (unless turned off by the usual git mechanism); this "
"flag specifies what do do with those refs afterward. "
"Replace refs can either be deleted or updated to point at new "
"commit hashes. Also, new replace refs can be added for each "
"commit rewrite. With 'update-or-add', new replace refs are "
"only added for commit rewrites that aren't used to update an "
"existing replace ref. default is 'update-and-add' if "
"$GIT_DIR/filter-repo/already_ran does not exist; "
"'update-or-add' otherwise."))
parents.add_argument('--prune-empty', default='auto',
choices=['always', 'auto', 'never'],
help=_("Whether to prune empty commits. 'auto' (the default) means "
"only prune commits which become empty (not commits which were "
"empty in the original repo, unless their parent was pruned). "
"When the parent of a commit is pruned, the first non-pruned "
"ancestor becomes the new parent."))
parents.add_argument('--prune-degenerate', default='auto',
choices=['always', 'auto', 'never'],
help=_("Since merge commits are needed for history topology, they "
"are typically exempt from pruning. However, they can become "
"degenerate with the pruning of other commits (having fewer "
"than two parents, having one commit serve as both parents, or "
"having one parent as the ancestor of the other.) If such "
"merge commits have no file changes, they can be pruned. The "
"default ('auto') is to only prune empty merge commits which "
"become degenerate (not which started as such)."))
parents.add_argument('--no-ff', action='store_true',
help=_("Even if the first parent is or becomes an ancestor of another "
"parent, do not prune it. This modifies how "
"--prune-degenerate behaves, and may be useful in projects who "
"always use merge --no-ff."))
callback = parser.add_argument_group(title=_("Generic callback code snippets"))
callback.add_argument('--filename-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing filenames; see CALLBACKS "
"sections below."))
callback.add_argument('--message-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing messages (both commit "
"messages and tag messages); see CALLBACKS section below."))
callback.add_argument('--name-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing names of people; see "
"CALLBACKS section below."))
callback.add_argument('--email-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing emails addresses; see "
"CALLBACKS section below."))
callback.add_argument('--refname-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing refnames; see CALLBACKS "
"section below."))
callback.add_argument('--blob-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing blob objects; see "
"CALLBACKS section below."))
callback.add_argument('--commit-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing commit objects; see "
"CALLBACKS section below."))
callback.add_argument('--tag-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing tag objects; see CALLBACKS "
"section below."))
callback.add_argument('--reset-callback', metavar="FUNCTION_BODY_OR_FILE",
help=_("Python code body for processing reset objects; see "
"CALLBACKS section below."))
desc = _(
"Specifying alternate source or target locations implies --partial,\n"
"except that the normal default for --replace-refs is used. However,\n"
"unlike normal uses of --partial, this doesn't risk mixing old and new\n"
"history since the old and new histories are in different repositories.")
location = parser.add_argument_group(title=_("Location to filter from/to"),
description=desc)
location.add_argument('--source', type=os.fsencode,
help=_("Git repository to read from"))
location.add_argument('--target', type=os.fsencode,
help=_("Git repository to overwrite with filtered history"))
misc = parser.add_argument_group(title=_("Miscellaneous options"))
misc.add_argument('--help', '-h', action='store_true',
help=_("Show this help message and exit."))
misc.add_argument('--version', action='store_true',
help=_("Display filter-repo's version and exit."))
misc.add_argument('--force', '-f', action='store_true',
help=_("Rewrite repository history even if the current repo does not "
"look like a fresh clone. History rewriting is irreversible "
"(and includes immediate pruning of reflogs and old objects), "
"so be cautious about using this flag."))
misc.add_argument('--partial', action='store_true',
help=_("Do a partial history rewrite, resulting in the mixture of "
"old and new history. This implies a default of "
"update-no-add for --replace-refs, disables rewriting "
"refs/remotes/origin/* to refs/heads/*, disables removing "
"of the 'origin' remote, disables removing unexported refs, "
"disables expiring the reflog, and disables the automatic "
"post-filter gc. Also, this modifies --tag-rename and "
"--refname-callback options such that instead of replacing "
"old refs with new refnames, it will instead create new "
"refs and keep the old ones around. Use with caution."))
# WARNING: --refs presents a problem with become-degenerate pruning:
# * Excluding a commit also excludes its ancestors so when some other
# commit has an excluded ancestor as a parent we have no way of
# knowing what it is an ancestor of without doing a special
# full-graph walk.
misc.add_argument('--refs', nargs='+',
help=_("Limit history rewriting to the specified refs. Implies "
"--partial. In addition to the normal caveats of --partial "
"(mixing old and new history, no automatic remapping of "
"refs/remotes/origin/* to refs/heads/*, etc.), this also may "
"cause problems for pruning of degenerate empty merge "
"commits when negative revisions are specified."))
misc.add_argument('--dry-run', action='store_true',
help=_("Do not change the repository. Run `git fast-export` and "
"filter its output, and save both the original and the "
"filtered version for comparison. This also disables "
"rewriting commit messages due to not knowing new commit "
"IDs and disables filtering of some empty commits due to "
"inability to query the fast-import backend." ))
misc.add_argument('--debug', action='store_true',
help=_("Print additional information about operations being "
"performed and commands being run. When used together "
"with --dry-run, also show extra information about what "
"would be run."))
# WARNING: --state-branch has some problems:
# * It does not work well with manually inserted objects (user creating
# Blob() or Commit() or Tag() objects and calling
# RepoFilter.insert(obj) on them).
# * It does not work well with multiple source or multiple target repos
# * It doesn't work so well with pruning become-empty commits (though
# --refs doesn't work so well with it either)
# These are probably fixable, given some work (e.g. re-importing the
# graph at the beginning to get the AncestryGraph right, doing our own
# export of marks instead of using fast-export --export-marks, etc.), but
# for now just hide the option.
misc.add_argument('--state-branch',
#help=_("Enable incremental filtering by saving the mapping of old "
# "to new objects to the specified branch upon exit, and"
# "loading that mapping from that branch (if it exists) "
# "upon startup."))
help=argparse.SUPPRESS)
misc.add_argument('--stdin', action='store_true',
help=_("Instead of running `git fast-export` and filtering its "
"output, filter the fast-export stream from stdin. The "
"stdin must be in the expected input format (e.g. it needs "
"to include original-oid directives)."))
misc.add_argument('--quiet', action='store_true',
help=_("Pass --quiet to other git commands called"))
return parser
@staticmethod
def sanity_check_args(args):
if args.analyze and args.path_changes:
raise SystemExit(_("Error: --analyze is incompatible with --path* flags; "
"it's a read-only operation."))
if args.analyze and args.stdin:
raise SystemExit(_("Error: --analyze is incompatible with --stdin."))
# If no path_changes are found, initialize with empty list but mark as
# not inclusive so that all files match
if args.path_changes == None:
args.path_changes = []
args.inclusive = False
else:
# Similarly, if we have no filtering paths, then no path should be
# filtered out. Based on how newname() works, the easiest way to
# achieve that is setting args.inclusive to False.
if not any(x[0] == 'filter' for x in args.path_changes):
args.inclusive = False
# Also check for incompatible --use-base-name and --path-rename flags.
if args.use_base_name:
if any(x[0] == 'rename' for x in args.path_changes):
raise SystemExit(_("Error: --use-base-name and --path-rename are "
"incompatible."))
# Also throw some sanity checks on git version here;
# PERF: remove these checks once new enough git versions are common
p = subproc.Popen('git fast-export -h'.split(),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = p.stdout.read()
if b'--anonymize-map' not in output: # pragma: no cover
global date_format_permissive
date_format_permissive = False
if b'--mark-tags' not in output: # pragma: no cover
global write_marks
write_marks = False
if args.state_branch:
# We need a version of git-fast-export with --mark-tags
raise SystemExit(_("Error: need git >= 2.24.0"))
if b'--reencode' not in output: # pragma: no cover
if args.preserve_commit_encoding:
# We need a version of git-fast-export with --reencode
raise SystemExit(_("Error: need git >= 2.23.0"))
else:
# Set args.preserve_commit_encoding to None which we'll check for later
# to avoid passing --reencode=yes to fast-export (that option was the
# default prior to git-2.23)
args.preserve_commit_encoding = None
# If we don't have fast-exoprt --reencode, we may also be missing
# diff-tree --combined-all-paths, which is even more important...
p = subproc.Popen('git diff-tree -h'.split(),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = p.stdout.read()
if b'--combined-all-paths' not in output:
# We need a version of git-diff-tree with --combined-all-paths
raise SystemExit(_("Error: need git >= 2.22.0"))
# End of sanity checks on git version
if args.max_blob_size:
suffix = args.max_blob_size[-1]
if suffix not in '1234567890':
mult = {'K': 1024, 'M': 1024**2, 'G': 1024**3}
if suffix not in mult:
raise SystemExit(_("Error: could not parse --strip-blobs-bigger-than"
" argument %s")
% args.max_blob_size)
args.max_blob_size = int(args.max_blob_size[0:-1]) * mult[suffix]
else:
args.max_blob_size = int(args.max_blob_size)
@staticmethod
def get_replace_text(filename):
replace_literals = []
replace_regexes = []
with open(filename, 'br') as f:
for line in f:
line = line.rstrip(b'\r\n')
# Determine the replacement
replacement = FilteringOptions.default_replace_text
if b'==>' in line:
line, replacement = line.rsplit(b'==>', 1)
# See if we need to match via regex
regex = None
if line.startswith(b'regex:'):
regex = line[6:]
elif line.startswith(b'glob:'):
regex = glob_to_regex(line[5:])
if regex:
replace_regexes.append((re.compile(regex), replacement))
else:
# Otherwise, find the literal we need to replace
if line.startswith(b'literal:'):
line = line[8:]
if not line:
continue
replace_literals.append((line, replacement))
return {'literals': replace_literals, 'regexes': replace_regexes}
@staticmethod
def get_paths_from_file(filename):
new_path_changes = []
with open(filename, 'br') as f:
for line in f:
line = line.rstrip(b'\r\n')
# Skip blank lines
if not line:
continue
# Skip comment lines
if line.startswith(b'#'):
continue
# Determine the replacement
match_type, repl = 'literal', None
if b'==>' in line:
line, repl = line.rsplit(b'==>', 1)
# See if we need to match via regex
match_type = 'match' # a.k.a. 'literal'
if line.startswith(b'regex:'):
match_type = 'regex'
match = re.compile(line[6:])
elif line.startswith(b'glob:'):
match_type = 'glob'
match = line[5:]
if repl:
raise SystemExit(_("Error: In %s, 'glob:' and '==>' are incompatible (renaming globs makes no sense)" % decode(filename)))
else:
if line.startswith(b'literal:'):
match = line[8:]
else:
match = line
if repl is not None:
if match and repl and match.endswith(b'/') != repl.endswith(b'/'):
raise SystemExit(_("Error: When rename directories, if OLDNAME "
"and NEW_NAME are both non-empty and either "
"ends with a slash then both must."))
# Record the filter or rename
if repl is not None:
new_path_changes.append(['rename', match_type, (match, repl)])
else:
new_path_changes.append(['filter', match_type, match])
if match_type == 'glob' and not match.endswith(b'*'):
extension = b'*' if match.endswith(b'/') else b'/*'
new_path_changes.append(['filter', match_type, match+extension])
return new_path_changes
@staticmethod
def default_options():
return FilteringOptions.parse_args([], error_on_empty = False)
@staticmethod
def parse_args(input_args, error_on_empty = True):
parser = FilteringOptions.create_arg_parser()
if not input_args and error_on_empty:
parser.print_usage()
raise SystemExit(_("No arguments specified."))
args = parser.parse_args(input_args)
if args.help:
parser.print_help()
raise SystemExit()
if args.version:
GitUtils.print_my_version()
raise SystemExit()
FilteringOptions.sanity_check_args(args)
if args.mailmap:
args.mailmap = MailmapInfo(args.mailmap)
if args.replace_text:
args.replace_text = FilteringOptions.get_replace_text(args.replace_text)
if args.replace_message:
args.replace_message = FilteringOptions.get_replace_text(args.replace_message)
if args.strip_blobs_with_ids:
with open(args.strip_blobs_with_ids, 'br') as f:
args.strip_blobs_with_ids = set(f.read().split())
else:
args.strip_blobs_with_ids = set()
if (args.partial or args.refs) and not args.replace_refs:
args.replace_refs = 'update-no-add'
args.repack = not (args.partial or args.refs)
if args.refs or args.source or args.target:
args.partial = True
if not args.refs:
args.refs = ['--all']
return args
class RepoAnalyze(object):
# First, several helper functions for analyze_commit()
@staticmethod
def equiv_class(stats, filename):
return stats['equivalence'].get(filename, (filename,))
@staticmethod
def setup_equivalence_for_rename(stats, oldname, newname):
# if A is renamed to B and B is renamed to C, then the user thinks of
# A, B, and C as all being different names for the same 'file'. We record
# this as an equivalence class:
# stats['equivalence'][name] = (A,B,C)
# for name being each of A, B, and C.
old_tuple = stats['equivalence'].get(oldname, ())
if newname in old_tuple:
return
elif old_tuple:
new_tuple = tuple(list(old_tuple)+[newname])
else:
new_tuple = (oldname, newname)
for f in new_tuple:
stats['equivalence'][f] = new_tuple
@staticmethod
def setup_or_update_rename_history(stats, commit, oldname, newname):
rename_commits = stats['rename_history'].get(oldname, set())
rename_commits.add(commit)
stats['rename_history'][oldname] = rename_commits
@staticmethod
def handle_renames(stats, commit, change_types, filenames):
for index, change_type in enumerate(change_types):
if change_type == ord(b'R'):
oldname, newname = filenames[index], filenames[-1]
RepoAnalyze.setup_equivalence_for_rename(stats, oldname, newname)
RepoAnalyze.setup_or_update_rename_history(stats, commit,
oldname, newname)
@staticmethod
def handle_file(stats, graph, commit, modes, shas, filenames):
mode, sha, filename = modes[-1], shas[-1], filenames[-1]
# Figure out kind of deletions to undo for this file, and update lists
# of all-names-by-sha and all-filenames
delmode = 'tree_deletions'
if mode != b'040000':
delmode = 'file_deletions'
stats['names'][sha].add(filename)
stats['allnames'].add(filename)
# If the file (or equivalence class of files) was recorded as deleted,
# clearly it isn't anymore
equiv = RepoAnalyze.equiv_class(stats, filename)
for f in equiv:
stats[delmode].pop(f, None)
# If we get a modify/add for a path that was renamed, we may need to break
# the equivalence class. However, if the modify/add was on a branch that
# doesn't have the rename in its history, we are still okay.
need_to_break_equivalence = False
if equiv[-1] != filename:
for rename_commit in stats['rename_history'][filename]:
if graph.is_ancestor(rename_commit, commit):
need_to_break_equivalence = True
if need_to_break_equivalence:
for f in equiv:
if f in stats['equivalence']:
del stats['equivalence'][f]
@staticmethod
def analyze_commit(stats, graph, commit, parents, date, file_changes):
graph.add_commit_and_parents(commit, parents)
for change in file_changes:
modes, shas, change_types, filenames = change
if len(parents) == 1 and change_types.startswith(b'R'):
change_types = b'R' # remove the rename score; we don't care
if modes[-1] == b'160000':
continue
elif modes[-1] == b'000000':
# Track when files/directories are deleted
for f in RepoAnalyze.equiv_class(stats, filenames[-1]):
if any(x == b'040000' for x in modes[0:-1]):
stats['tree_deletions'][f] = date
else:
stats['file_deletions'][f] = date
elif change_types.strip(b'AMT') == b'':
RepoAnalyze.handle_file(stats, graph, commit, modes, shas, filenames)
elif modes[-1] == b'040000' and change_types.strip(b'RAM') == b'':
RepoAnalyze.handle_file(stats, graph, commit, modes, shas, filenames)
elif change_types.strip(b'RAMT') == b'':
RepoAnalyze.handle_file(stats, graph, commit, modes, shas, filenames)
RepoAnalyze.handle_renames(stats, commit, change_types, filenames)
else:
raise SystemExit(_("Unhandled change type(s): %(change_type)s "
"(in commit %(commit)s)")
% ({'change_type': change_types, 'commit': commit})
) # pragma: no cover
@staticmethod
def gather_data(args):
unpacked_size, packed_size = GitUtils.get_blob_sizes()
stats = {'names': collections.defaultdict(set),
'allnames' : set(),
'file_deletions': {},
'tree_deletions': {},
'equivalence': {},
'rename_history': collections.defaultdict(set),
'unpacked_size': unpacked_size,
'packed_size': packed_size,
'num_commits': 0}
# Setup the rev-list/diff-tree process
processed_commits_msg = _("Processed %d commits")
commit_parse_progress = ProgressWriter()
num_commits = 0
cmd = ('git rev-list --topo-order --reverse {}'.format(' '.join(args.refs)) +
' | git diff-tree --stdin --always --root --format=%H%n%P%n%cd' +
' --date=short -M -t -c --raw --combined-all-paths')
dtp = subproc.Popen(cmd, shell=True, bufsize=-1, stdout=subprocess.PIPE)
f = dtp.stdout
line = f.readline()
if not line:
raise SystemExit(_("Nothing to analyze; repository is empty."))
cont = bool(line)
graph = AncestryGraph()
while cont:
commit = line.rstrip()
parents = f.readline().split()
date = f.readline().rstrip()
# We expect a blank line next; if we get a non-blank line then
# this commit modified no files and we need to move on to the next.
# If there is no line, we've reached end-of-input.
line = f.readline()
if not line:
cont = False
line = line.rstrip()
# If we haven't reached end of input, and we got a blank line meaning
# a commit that has modified files, then get the file changes associated
# with this commit.
file_changes = []
if cont and not line:
cont = False
for line in f:
if not line.startswith(b':'):
cont = True
break
n = 1+max(1, len(parents))
assert line.startswith(b':'*(n-1))
relevant = line[n-1:-1]
splits = relevant.split(None, n)
modes = splits[0:n]
splits = splits[n].split(None, n)
shas = splits[0:n]
splits = splits[n].split(b'\t')
change_types = splits[0]
filenames = [PathQuoting.dequote(x) for x in splits[1:]]
file_changes.append([modes, shas, change_types, filenames])
# If someone is trying to analyze a subset of the history, make sure
# to avoid dying on commits with parents that we haven't seen before
if args.refs:
graph.record_external_commits([p for p in parents
if not p in graph.value])
# Analyze this commit and update progress
RepoAnalyze.analyze_commit(stats, graph, commit, parents, date,
file_changes)
num_commits += 1
commit_parse_progress.show(processed_commits_msg % num_commits)
# Show the final commits processed message and record the number of commits
commit_parse_progress.finish()
stats['num_commits'] = num_commits
# Close the output, ensure rev-list|diff-tree pipeline completed successfully
dtp.stdout.close()
if dtp.wait():
raise SystemExit(_("Error: rev-list|diff-tree pipeline failed; see above.")) # pragma: no cover
return stats
@staticmethod
def write_report(reportdir, stats):
def datestr(datetimestr):
return datetimestr if datetimestr else _('<present>').encode()
def dirnames(path):
while True:
path = os.path.dirname(path)
yield path
if path == b'':
break
# Compute aggregate size information for paths, extensions, and dirs
total_size = {'packed': 0, 'unpacked': 0}
path_size = {'packed': collections.defaultdict(int),
'unpacked': collections.defaultdict(int)}
ext_size = {'packed': collections.defaultdict(int),
'unpacked': collections.defaultdict(int)}
dir_size = {'packed': collections.defaultdict(int),
'unpacked': collections.defaultdict(int)}
for sha in stats['names']:
size = {'packed': stats['packed_size'][sha],
'unpacked': stats['unpacked_size'][sha]}
for which in ('packed', 'unpacked'):
for name in stats['names'][sha]:
total_size[which] += size[which]
path_size[which][name] += size[which]
basename, ext = os.path.splitext(name)
ext_size[which][ext] += size[which]
for dirname in dirnames(name):
dir_size[which][dirname] += size[which]
# Determine if and when extensions and directories were deleted
ext_deleted_data = {}
for name in stats['allnames']:
when = stats['file_deletions'].get(name, None)
# Update the extension
basename, ext = os.path.splitext(name)
if when is None:
ext_deleted_data[ext] = None
elif ext in ext_deleted_data:
if ext_deleted_data[ext] is not None:
ext_deleted_data[ext] = max(ext_deleted_data[ext], when)
else:
ext_deleted_data[ext] = when
dir_deleted_data = {}
for name in dir_size['packed']:
dir_deleted_data[name] = stats['tree_deletions'].get(name, None)
with open(os.path.join(reportdir, b"README"), 'bw') as f:
# Give a basic overview of this file
f.write(b"== %s ==\n" % _("Overall Statistics").encode())
f.write((" %s: %d\n" % (_("Number of commits"),
stats['num_commits'])).encode())
f.write((" %s: %d\n" % (_("Number of filenames"),
len(path_size['packed']))).encode())
f.write((" %s: %d\n" % (_("Number of directories"),
len(dir_size['packed']))).encode())
f.write((" %s: %d\n" % (_("Number of file extensions"),
len(ext_size['packed']))).encode())
f.write(b"\n")
f.write((" %s: %d\n" % (_("Total unpacked size (bytes)"),
total_size['unpacked'])).encode())
f.write((" %s: %d\n" % (_("Total packed size (bytes)"),
total_size['packed'])).encode())
f.write(b"\n")
# Mention issues with the report
f.write(("== %s ==\n" % _("Caveats")).encode())
f.write(("=== %s ===\n" % _("Sizes")).encode())
f.write(textwrap.dedent(_("""
Packed size represents what size your repository would be if no
trees, commits, tags, or other metadata were included (though it may
fail to represent de-duplication; see below). It also represents the
current packing, which may be suboptimal if you haven't gc'ed for a
while.
Unpacked size represents what size your repository would be if no
trees, commits, tags, or other metadata were included AND if no
files were packed; i.e., without delta-ing or compression.
Both unpacked and packed sizes can be slightly misleading. Deleting
a blob from history not save as much space as the unpacked size,
because it is obviously normally stored in packed form. Also,
deleting a blob from history may not save as much space as its packed
size either, because another blob could be stored as a delta against
that blob, so when you remove one blob another blob's packed size may
grow.
Also, the sum of the packed sizes can add up to more than the
repository size; if the same contents appeared in the repository in
multiple places, git will automatically de-dupe and store only one
copy, while the way sizes are added in this analysis adds the size
for each file path that has those contents. Further, if a file is
ever reverted to a previous version's contents, the previous
version's size will be counted multiple times in this analysis, even
though git will only store it once.
""")[1:]).encode())
f.write(b"\n")
f.write(("=== %s ===\n" % _("Deletions")).encode())
f.write(textwrap.dedent(_("""
Whether a file is deleted is not a binary quality, since it can be
deleted on some branches but still exist in others. Also, it might
exist in an old tag, but have been deleted in versions newer than
that. More thorough tracking could be done, including looking at
merge commits where one side of history deleted and the other modified,
in order to give a more holistic picture of deletions. However, that
algorithm would not only be more complex to implement, it'd also be
quite difficult to present and interpret by users. Since --analyze
is just about getting a high-level rough picture of history, it instead
implements the simplistic rule that is good enough for 98% of cases:
A file is marked as deleted if the last commit in the fast-export
stream that mentions the file lists it as deleted.
This makes it dependent on topological ordering, but generally gives
the "right" answer.
""")[1:]).encode())
f.write(b"\n")
f.write(("=== %s ===\n" % _("Renames")).encode())
f.write(textwrap.dedent(_("""
Renames share the same non-binary nature that deletions do, plus
additional challenges:
* If the renamed file is renamed again, instead of just two names for
a path you can have three or more.
* Rename pairs of the form (oldname, newname) that we consider to be
different names of the "same file" might only be valid over certain
commit ranges. For example, if a new commit reintroduces a file
named oldname, then new versions of oldname aren't the "same file"
anymore. We could try to portray this to the user, but it's easier
for the user to just break the pairing and only report unbroken
rename pairings to the user.
* The ability for users to rename files differently in different
branches means that our chains of renames will not necessarily be
linear but may branch out.
""")[1:]).encode())
f.write(b"\n")
# Equivalence classes for names, so if folks only want to keep a
# certain set of paths, they know the old names they want to include
# too.
with open(os.path.join(reportdir, b"renames.txt"), 'bw') as f:
seen = set()
for pathname,equiv_group in sorted(stats['equivalence'].items(),
key=lambda x:(x[1], x[0])):
if equiv_group in seen:
continue
seen.add(equiv_group)
f.write(("{} ->\n ".format(decode(equiv_group[0])) +
"\n ".join(decode(x) for x in equiv_group[1:]) +
"\n").encode())
# List directories in reverse sorted order of unpacked size
with open(os.path.join(reportdir, b"directories-deleted-sizes.txt"), 'bw') as f:
msg = "=== %s ===\n" % _("Deleted directories by reverse size")
f.write(msg.encode())
msg = _("Format: unpacked size, packed size, date deleted, directory name\n")
f.write(msg.encode())
for dirname, size in sorted(dir_size['packed'].items(),
key=lambda x:(x[1],x[0]), reverse=True):
if (dir_deleted_data[dirname]):
f.write(b" %10d %10d %-10s %s\n" % (dir_size['unpacked'][dirname],
size,
datestr(dir_deleted_data[dirname]),
dirname or _('<toplevel>').encode()))
with open(os.path.join(reportdir, b"directories-all-sizes.txt"), 'bw') as f:
f.write(("=== %s ===\n" % _("All directories by reverse size")).encode())
msg = _("Format: unpacked size, packed size, date deleted, directory name\n")
f.write(msg.encode())
for dirname, size in sorted(dir_size['packed'].items(),
key=lambda x:(x[1],x[0]), reverse=True):
f.write(b" %10d %10d %-10s %s\n" % (dir_size['unpacked'][dirname],
size,
datestr(dir_deleted_data[dirname]),
dirname or _("<toplevel>").encode()))
# List extensions in reverse sorted order of unpacked size
with open(os.path.join(reportdir, b"extensions-deleted-sizes.txt"), 'bw') as f:
msg = "=== %s ===\n" % _("Deleted extensions by reverse size")
f.write(msg.encode())
msg = _("Format: unpacked size, packed size, date deleted, extension name\n")
f.write(msg.encode())
for extname, size in sorted(ext_size['packed'].items(),
key=lambda x:(x[1],x[0]), reverse=True):
if (ext_deleted_data[extname]):
f.write(b" %10d %10d %-10s %s\n" % (ext_size['unpacked'][extname],
size,
datestr(ext_deleted_data[extname]),
extname or _('<no extension>').encode()))
with open(os.path.join(reportdir, b"extensions-all-sizes.txt"), 'bw') as f:
f.write(("=== %s ===\n" % _("All extensions by reverse size")).encode())
msg = _("Format: unpacked size, packed size, date deleted, extension name\n")
f.write(msg.encode())
for extname, size in sorted(ext_size['packed'].items(),
key=lambda x:(x[1],x[0]), reverse=True):
f.write(b" %10d %10d %-10s %s\n" % (ext_size['unpacked'][extname],
size,
datestr(ext_deleted_data[extname]),
extname or _('<no extension>').encode()))
# List files in reverse sorted order of unpacked size
with open(os.path.join(reportdir, b"path-deleted-sizes.txt"), 'bw') as f:
msg = "=== %s ===\n" % _("Deleted paths by reverse accumulated size")
f.write(msg.encode())
msg = _("Format: unpacked size, packed size, date deleted, path name(s)\n")
f.write(msg.encode())
for pathname, size in sorted(path_size['packed'].items(),
key=lambda x:(x[1],x[0]), reverse=True):
when = stats['file_deletions'].get(pathname, None)
if when:
f.write(b" %10d %10d %-10s %s\n" % (path_size['unpacked'][pathname],
size,
datestr(when),
pathname))
with open(os.path.join(reportdir, b"path-all-sizes.txt"), 'bw') as f:
msg = "=== %s ===\n" % _("All paths by reverse accumulated size")
f.write(msg.encode())
msg = _("Format: unpacked size, packed size, date deleted, path name\n")
f.write(msg.encode())
for pathname, size in sorted(path_size['packed'].items(),
key=lambda x:(x[1],x[0]), reverse=True):
when = stats['file_deletions'].get(pathname, None)
f.write(b" %10d %10d %-10s %s\n" % (path_size['unpacked'][pathname],
size,
datestr(when),
pathname))
# List of filenames and sizes in descending order
with open(os.path.join(reportdir, b"blob-shas-and-paths.txt"), 'bw') as f:
f.write(("=== %s ===\n" % _("Files by sha and associated pathnames in reverse size")).encode())
f.write(_("Format: sha, unpacked size, packed size, filename(s) object stored as\n").encode())
for sha, size in sorted(stats['packed_size'].items(),
key=lambda x:(x[1],x[0]), reverse=True):
if sha not in stats['names']:
# Some objects in the repository might not be referenced, or not
# referenced by the branches/tags the user cares about; skip them.
continue
names_with_sha = stats['names'][sha]
if len(names_with_sha) == 1:
names_with_sha = names_with_sha.pop()
else:
names_with_sha = b'[' + b', '.join(sorted(names_with_sha)) + b']'
f.write(b" %s %10d %10d %s\n" % (sha,
stats['unpacked_size'][sha],
size,
names_with_sha))
@staticmethod
def run(args):
if args.report_dir:
reportdir = args.report_dir
else:
git_dir = GitUtils.determine_git_dir(b'.')
# Create the report directory as necessary
results_tmp_dir = os.path.join(git_dir, b'filter-repo')
if not os.path.isdir(results_tmp_dir):
os.mkdir(results_tmp_dir)
reportdir = os.path.join(results_tmp_dir, b"analysis")
if os.path.isdir(reportdir):
if args.force:
sys.stdout.write(_("Warning: Removing recursively: \"%s\"") % decode(reportdir))
shutil.rmtree(reportdir)
else:
sys.stdout.write(_("Error: dir already exists (use --force to delete): \"%s\"\n") % decode(reportdir))
sys.exit(1)
os.mkdir(reportdir)
# Gather the data we need
stats = RepoAnalyze.gather_data(args)
# Write the reports
sys.stdout.write(_("Writing reports to %s...") % decode(reportdir))
sys.stdout.flush()
RepoAnalyze.write_report(reportdir, stats)
sys.stdout.write(_("done.\n"))
class InputFileBackup:
def __init__(self, input_file, output_file):
self.input_file = input_file
self.output_file = output_file
def close(self):
self.input_file.close()
self.output_file.close()
def read(self, size):
output = self.input_file.read(size)
self.output_file.write(output)
return output
def readline(self):
line = self.input_file.readline()
self.output_file.write(line)
return line
class DualFileWriter:
def __init__(self, file1, file2):
self.file1 = file1
self.file2 = file2
def write(self, *args):
self.file1.write(*args)
self.file2.write(*args)
def flush(self):
self.file1.flush()
self.file2.flush()
def close(self):
self.file1.close()
self.file2.close()
class RepoFilter(object):
def __init__(self,
args,
filename_callback = None,
message_callback = None,
name_callback = None,
email_callback = None,
refname_callback = None,
blob_callback = None,
commit_callback = None,
tag_callback = None,
reset_callback = None,
done_callback = None):
self._args = args
# Repo we are exporting
self._repo_working_dir = None
# Store callbacks for acting on objects printed by FastExport
self._blob_callback = blob_callback
self._commit_callback = commit_callback
self._tag_callback = tag_callback
self._reset_callback = reset_callback
self._done_callback = done_callback
# Store callbacks for acting on slices of FastExport objects
self._filename_callback = filename_callback # filenames from commits
self._message_callback = message_callback # commit OR tag message
self._name_callback = name_callback # author, committer, tagger
self._email_callback = email_callback # author, committer, tagger
self._refname_callback = refname_callback # from commit/tag/reset
self._handle_arg_callbacks()
# Defaults for input
self._input = None
self._fep = None # Fast Export Process
self._fe_orig = None # Path to where original fast-export output stored
self._fe_filt = None # Path to where filtered fast-export output stored
self._parser = None # FastExportParser object we are working with
# Defaults for output
self._output = None
self._fip = None # Fast Import Process
self._import_pipes = None
self._managed_output = True
# A tuple of (depth, list-of-ancestors). Commits and ancestors are
# identified by their id (their 'mark' in fast-export or fast-import
# speak). The depth of a commit is one more than the max depth of any
# of its ancestors.
self._graph = AncestryGraph()
# Another one, for ancestry of commits in the original repo
self._orig_graph = AncestryGraph()
# Names of files that were tweaked in any commit; such paths could lead
# to subsequent commits being empty
self._files_tweaked = set()
# A set of commit hash pairs (oldhash, newhash) which used to be merge
# commits but due to filtering were turned into non-merge commits.
# The commits probably have suboptimal commit messages (e.g. "Merge branch
# next into master").
self._commits_no_longer_merges = []
# A dict of original_ids to new_ids; filtering commits means getting
# new commit hash (sha1sums), and we record the mapping both for
# diagnostic purposes and so we can rewrite commit messages. Note that
# the new_id can be None rather than a commit hash if the original
# commit became empty and was pruned or was otherwise dropped.
self._commit_renames = {}
# A set of original_ids for which we have not yet gotten the
# new_ids; we use OrderedDict because we need to know the order of
# insertion, but the values are always ignored (and set to None).
# If there was an OrderedSet class, I'd use it instead.
self._pending_renames = collections.OrderedDict()
# A dict of commit_hash[0:7] -> set(commit_hashes with that prefix).
#
# It's common for commit messages to refer to commits by abbreviated
# commit hashes, as short as 7 characters. To facilitate translating
# such short hashes, we have a mapping of prefixes to full old hashes.
self._commit_short_old_hashes = collections.defaultdict(set)
# A set of commit hash references appearing in commit messages which
# mapped to a valid commit that was removed entirely in the filtering
# process. The commit message will continue to reference the
# now-missing commit hash, since there was nothing to map it to.
self._commits_referenced_but_removed = set()
# Progress handling (number of commits parsed, etc.)
self._progress_writer = ProgressWriter()
self._num_commits = 0
# Size of blobs in the repo
self._unpacked_size = {}
# Other vars
self._sanity_checks_handled = False
self._finalize_handled = False
self._orig_refs = None
self._newnames = {}
# Cache a few message translations for performance reasons
self._parsed_message = _("Parsed %d commits")
# Compile some regexes and cache those
self._hash_re = re.compile(br'(\b[0-9a-f]{7,40}\b)')
def _handle_arg_callbacks(self):
def make_callback(argname, str):
exec('def callback({}, _do_not_use_this_var = None):\n'.format(argname)+
' '+'\n '.join(str.splitlines()), globals())
return callback #namespace['callback']
def handle(type):
callback_field = '_{}_callback'.format(type)
code_string = getattr(self._args, type+'_callback')
if code_string:
if os.path.exists(code_string):
with open(code_string, 'r', encoding='utf-8') as f:
code_string = f.read()
if getattr(self, callback_field):
raise SystemExit(_("Error: Cannot pass a %s_callback to RepoFilter "
"AND pass --%s-callback"
% (type, type)))
if 'return ' not in code_string and \
type not in ('blob', 'commit', 'tag', 'reset'):
raise SystemExit(_("Error: --%s-callback should have a return statement")
% type)
setattr(self, callback_field, make_callback(type, code_string))
handle('filename')
handle('message')
handle('name')
handle('email')
handle('refname')
handle('blob')
handle('commit')
handle('tag')
handle('reset')
def _run_sanity_checks(self):
self._sanity_checks_handled = True
if not self._managed_output:
if not self._args.replace_refs:
# If not _managed_output we don't want to make extra changes to the
# repo, so set default to no-op 'update-no-add'
self._args.replace_refs = 'update-no-add'
return
if self._args.debug:
print("[DEBUG] Passed arguments:\n{}".format(self._args))
# Determine basic repository information
target_working_dir = self._args.target or b'.'
self._orig_refs = GitUtils.get_refs(target_working_dir)
is_bare = GitUtils.is_repository_bare(target_working_dir)
# Determine if this is second or later run of filter-repo
tmp_dir = self.results_tmp_dir(create_if_missing=False)
already_ran = os.path.isfile(os.path.join(tmp_dir, b'already_ran'))
# Default for --replace-refs
if not self._args.replace_refs:
self._args.replace_refs = ('update-or-add' if already_ran
else 'update-and-add')
# Do sanity checks from the correct directory
if not self._args.force and not already_ran:
cwd = os.getcwd()
os.chdir(target_working_dir)
RepoFilter.sanity_check(self._orig_refs, is_bare)
os.chdir(cwd)
@staticmethod
def sanity_check(refs, is_bare):
def abort(reason):
try:
cmd = 'git config remote.origin.url'
output = subproc.check_output(cmd.split()).strip()
except subprocess.CalledProcessError as e:
output = None
msg = ""
if output and os.path.isdir(output):
msg = _("Note: when cloning local repositories, you need to pass\n"
" --no-local to git clone to avoid this issue.\n")
raise SystemExit(
_("Aborting: Refusing to destructively overwrite repo history since\n"
"this does not look like a fresh clone.\n"
" (%s)\n%s"
"Please operate on a fresh clone instead. If you want to proceed\n"
"anyway, use --force.") % (reason, msg))
# Make sure repo is fully packed, just like a fresh clone would be.
# Note that transfer.unpackLimit defaults to 100, meaning that a
# repository with no packs and less than 100 objects should be considered
# fully packed.
output = subproc.check_output('git count-objects -v'.split())
stats = dict(x.split(b': ') for x in output.splitlines())
num_packs = int(stats[b'packs'])
num_loose_objects = int(stats[b'count'])
if num_packs > 1 or \
(num_packs == 1 and num_loose_objects > 0) or \
num_loose_objects >= 100:
abort(_("expected freshly packed repo"))
# Make sure there is precisely one remote, named "origin"...or that this
# is a new bare repo with no packs and no remotes
output = subproc.check_output('git remote'.split()).strip()
if not (output == b"origin" or (num_packs == 0 and not output)):
abort(_("expected one remote, origin"))
# Avoid letting people running with weird setups and overwriting GIT_DIR
# elsewhere
git_dir = GitUtils.determine_git_dir(b'.')
if is_bare and git_dir != b'.':
abort(_("GIT_DIR must be ."))
elif not is_bare and git_dir != b'.git':
abort(_("GIT_DIR must be .git"))
# Make sure that all reflogs have precisely one entry
reflog_dir=os.path.join(git_dir, b'logs')
for root, dirs, files in os.walk(reflog_dir):
for filename in files:
pathname = os.path.join(root, filename)
with open(pathname, 'br') as f:
if len(f.read().splitlines()) > 1:
shortpath = pathname[len(reflog_dir)+1:]
abort(_("expected at most one entry in the reflog for %s") %
decode(shortpath))
# Make sure there are no stashed changes
if b'refs/stash' in refs:
abort(_("has stashed changes"))
# Do extra checks in non-bare repos
if not is_bare:
# Avoid uncommitted, unstaged, or untracked changes
if subproc.call('git diff --staged --quiet'.split()):
abort(_("you have uncommitted changes"))
if subproc.call('git diff --quiet'.split()):
abort(_("you have unstaged changes"))
if len(subproc.check_output('git ls-files -o'.split())) > 0:
abort(_("you have untracked changes"))
# Avoid unpushed changes
for refname, rev in refs.items():
if not refname.startswith(b'refs/heads/'):
continue
origin_ref = refname.replace(b'refs/heads/', b'refs/remotes/origin/')
if origin_ref not in refs:
abort(_('%s exists, but %s not found') % (decode(refname),
decode(origin_ref)))
if rev != refs[origin_ref]:
abort(_('%s does not match %s') % (decode(refname),
decode(origin_ref)))
# Make sure there is only one worktree
output = subproc.check_output('git worktree list'.split())
if len(output.splitlines()) > 1:
abort(_('you have multiple worktrees'))
@staticmethod
def cleanup(repo, repack, reset, run_quietly=False, show_debuginfo=False):
''' cleanup repo; if repack then expire reflogs and do a gc --prune=now.
if reset then do a reset --hard. Optionally also curb output if
run_quietly is True, or go the opposite direction and show extra
output if show_debuginfo is True. '''
assert not (run_quietly and show_debuginfo)
if (repack and not run_quietly and not show_debuginfo):
print(_("Repacking your repo and cleaning out old unneeded objects"))
quiet_flags = '--quiet' if run_quietly else ''
cleanup_cmds = []
if repack:
cleanup_cmds = ['git reflog expire --expire=now --all'.split(),
'git gc {} --prune=now'.format(quiet_flags).split()]
if reset:
cleanup_cmds.insert(0, 'git reset {} --hard'.format(quiet_flags).split())
location_info = ' (in {})'.format(decode(repo)) if repo != b'.' else ''
for cmd in cleanup_cmds:
if show_debuginfo:
print("[DEBUG] Running{}: {}".format(location_info, ' '.join(cmd)))
subproc.call(cmd, cwd=repo)
def _get_rename(self, old_hash):
# If we already know the rename, just return it
new_hash = self._commit_renames.get(old_hash, None)
if new_hash:
return new_hash
# If it's not in the remaining pending renames, we don't know it
if old_hash is not None and old_hash not in self._pending_renames:
return None
# Read through the pending renames until we find it or we've read them all,
# and return whatever we might find
self._flush_renames(old_hash)
return self._commit_renames.get(old_hash, None)
def _flush_renames(self, old_hash=None, limit=0):
# Parse through self._pending_renames until we have read enough. We have
# read enough if:
# self._pending_renames is empty
# old_hash != None and we found a rename for old_hash
# limit > 0 and len(self._pending_renames) started less than 2*limit
# limit > 0 and len(self._pending_renames) < limit
if limit and len(self._pending_renames) < 2 * limit:
return
fi_input, fi_output = self._import_pipes
while self._pending_renames:
orig_id, ignore = self._pending_renames.popitem(last=False)
new_id = fi_output.readline().rstrip()
self._commit_renames[orig_id] = new_id
if old_hash == orig_id:
return
if limit and len(self._pending_renames) < limit:
return
def _translate_commit_hash(self, matchobj_or_oldhash):
old_hash = matchobj_or_oldhash
if not isinstance(matchobj_or_oldhash, bytes):
old_hash = matchobj_or_oldhash.group(1)
orig_len = len(old_hash)
new_hash = self._get_rename(old_hash)
if new_hash is None:
if old_hash[0:7] not in self._commit_short_old_hashes:
self._commits_referenced_but_removed.add(old_hash)
return old_hash
possibilities = self._commit_short_old_hashes[old_hash[0:7]]
matches = [x for x in possibilities
if x[0:orig_len] == old_hash]
if len(matches) != 1:
self._commits_referenced_but_removed.add(old_hash)
return old_hash
old_hash = matches[0]
new_hash = self._get_rename(old_hash)
assert new_hash is not None
return new_hash[0:orig_len]
def _trim_extra_parents(self, orig_parents, parents):
'''Due to pruning of empty commits, some parents could be non-existent
(None) or otherwise redundant. Remove the non-existent parents, and
remove redundant parents so long as that doesn't transform a merge
commit into a non-merge commit.
Returns a tuple:
(parents, new_first_parent_if_would_become_non_merge)'''
always_prune = (self._args.prune_degenerate == 'always')
# Pruning of empty commits means multiple things:
# * An original parent of this commit may have been pruned causing the
# need to rewrite the reported parent to the nearest ancestor. We
# want to know when we're dealing with such a parent.
# * Further, there may be no "nearest ancestor" if the entire history
# of that parent was also pruned. (Detectable by the parent being
# 'None')
# Remove all parents rewritten to None, and keep track of which parents
# were rewritten to an ancestor.
tmp = zip(parents,
orig_parents,
[(x in _SKIPPED_COMMITS or always_prune) for x in orig_parents])
tmp2 = [x for x in tmp if x[0] is not None]
if not tmp2:
# All ancestors have been pruned; we have no parents.
return [], None
parents, orig_parents, is_rewritten = [list(x) for x in zip(*tmp2)]
# We can't have redundant parents if we don't have at least 2 parents
if len(parents) < 2:
return parents, None
# Don't remove redundant parents if user doesn't want us to
if self._args.prune_degenerate == 'never':
return parents, None
# Remove duplicate parents (if both sides of history have lots of commits
# which become empty due to pruning, the most recent ancestor on both
# sides may be the same commit), except only remove parents that have
# been rewritten due to previous empty pruning.
seen = set()
seen_add = seen.add
# Deleting duplicate rewritten parents means keeping parents if either
# they have not been seen or they are ones that have not been rewritten.
parents_copy = parents
uniq = [[p, orig_parents[i], is_rewritten[i]] for i, p in enumerate(parents)
if not (p in seen or seen_add(p)) or not is_rewritten[i]]
parents, orig_parents, is_rewritten = [list(x) for x in zip(*uniq)]
if len(parents) < 2:
return parents_copy, parents[0]
# Flatten unnecessary merges. (If one side of history is entirely
# empty commits that were pruned, we may end up attempting to
# merge a commit with its ancestor. Remove parents that are an
# ancestor of another parent.)
num_parents = len(parents)
to_remove = []
for cur in range(num_parents):
if not is_rewritten[cur]:
continue
for other in range(num_parents):
if cur == other:
continue
if not self._graph.is_ancestor(parents[cur], parents[other]):
continue
# parents[cur] is an ancestor of parents[other], so parents[cur]
# seems redundant. However, if it was intentionally redundant
# (e.g. a no-ff merge) in the original, then we want to keep it.
if not always_prune and \
self._orig_graph.is_ancestor(orig_parents[cur],
orig_parents[other]):
continue
# Some folks want their history to have all first parents be merge
# commits (except for any root commits), and always do a merge --no-ff.
# For such folks, don't remove the first parent even if it's an
# ancestor of other commits.
if self._args.no_ff and cur == 0:
continue
# Okay so the cur-th parent is an ancestor of the other-th parent,
# and it wasn't that way in the original repository; mark the
# cur-th parent as removable.
to_remove.append(cur)
break # cur removed, so skip rest of others -- i.e. check cur+=1
for x in reversed(to_remove):
parents.pop(x)
if len(parents) < 2:
return parents_copy, parents[0]
return parents, None
def _prunable(self, commit, new_1st_parent, had_file_changes, orig_parents):
parents = commit.parents
if self._args.prune_empty == 'never':
return False
always_prune = (self._args.prune_empty == 'always')
# For merge commits, unless there are prunable (redundant) parents, we
# do not want to prune
if len(parents) >= 2 and not new_1st_parent:
return False
if len(parents) < 2:
# Special logic for commits that started empty...
if not had_file_changes and not always_prune:
had_parents_pruned = (len(parents) < len(orig_parents) or
(len(orig_parents) == 1 and
orig_parents[0] in _SKIPPED_COMMITS))
# If the commit remains empty and had parents which were pruned,
# then prune this commit; otherwise, retain it
return (not commit.file_changes and had_parents_pruned)
# We can only get here if the commit didn't start empty, so if it's
# empty now, it obviously became empty
if not commit.file_changes:
return True
# If there are no parents of this commit and we didn't match the case
# above, then this commit cannot be pruned. Since we have no parent(s)
# to compare to, abort now to prevent future checks from failing.
if not parents:
return False
# Similarly, we cannot handle the hard cases if we don't have a pipe
# to communicate with fast-import
if not self._import_pipes:
return False
# If there have not been renames/remappings of IDs (due to insertion of
# new blobs), then we can sometimes know things aren't prunable with a
# simple check
if not _IDS.has_renames():
# non-merge commits can only be empty if blob/file-change editing caused
# all file changes in the commit to have the same file contents as
# the parent.
changed_files = set(change.filename for change in commit.file_changes)
if len(orig_parents) < 2 and changed_files - self._files_tweaked:
return False
# Finally, the hard case: due to either blob rewriting, or due to pruning
# of empty commits wiping out the first parent history back to the merge
# base, the list of file_changes we have may not actually differ from our
# (new) first parent's version of the files, i.e. this would actually be
# an empty commit. Check by comparing the contents of this commit to its
# (remaining) parent.
#
# NOTE on why this works, for the case of original first parent history
# having been pruned away due to being empty:
# The first parent history having been pruned away due to being
# empty implies the original first parent would have a tree (after
# filtering) that matched the merge base's tree. Since
# file_changes has the changes needed to go from what would have
# been the first parent to our new commit, and what would have been
# our first parent has a tree that matches the merge base, then if
# the new first parent has a tree matching the versions of files in
# file_changes, then this new commit is empty and thus prunable.
fi_input, fi_output = self._import_pipes
self._flush_renames() # Avoid fi_output having other stuff present
# Optimization note: we could have two loops over file_changes, the
# first doing all the self._output.write() calls, and the second doing
# the rest. But I'm worried about fast-import blocking on fi_output
# buffers filling up so I instead read from it as I go.
for change in commit.file_changes:
parent = new_1st_parent or commit.parents[0] # exists due to above checks
quoted_filename = PathQuoting.enquote(change.filename)
if isinstance(parent, int):
self._output.write(b"ls :%d %s\n" % (parent, quoted_filename))
else:
self._output.write(b"ls %s %s\n" % (parent, quoted_filename))
self._output.flush()
parent_version = fi_output.readline().split()
if change.type == b'D':
if parent_version != [b'missing', quoted_filename]:
return False
else:
blob_sha = change.blob_id
if isinstance(change.blob_id, int):
self._output.write(b"get-mark :%d\n" % change.blob_id)
self._output.flush()
blob_sha = fi_output.readline().rstrip()
if parent_version != [change.mode, b'blob', blob_sha, quoted_filename]:
return False
return True
def _record_remapping(self, commit, orig_parents):
new_id = None
# Record the mapping of old commit hash to new one
if commit.original_id and self._import_pipes:
fi_input, fi_output = self._import_pipes
self._output.write(b"get-mark :%d\n" % commit.id)
self._output.flush()
orig_id = commit.original_id
self._commit_short_old_hashes[orig_id[0:7]].add(orig_id)
# Note that we have queued up an id for later reading; flush a
# few of the older ones if we have too many queued up
self._pending_renames[orig_id] = None
self._flush_renames(None, limit=40)
# Also, record if this was a merge commit that turned into a non-merge
# commit.
if len(orig_parents) >= 2 and len(commit.parents) < 2:
self._commits_no_longer_merges.append((commit.original_id, new_id))
def callback_metadata(self, extra_items = dict()):
return {'commit_rename_func': self._translate_commit_hash,
'ancestry_graph': self._graph,
'original_ancestry_graph': self._orig_graph,
**extra_items}
def _tweak_blob(self, blob):
if self._args.max_blob_size and len(blob.data) > self._args.max_blob_size:
blob.skip()
if blob.original_id in self._args.strip_blobs_with_ids:
blob.skip()
if ( self._args.replace_text
# not (if blob contains zero byte in the first 8Kb, that is, if blob is binary data)
and not b"\0" in blob.data[0:8192]
):
for literal, replacement in self._args.replace_text['literals']:
blob.data = blob.data.replace(literal, replacement)
for regex, replacement in self._args.replace_text['regexes']:
blob.data = regex.sub(replacement, blob.data)
if self._blob_callback:
self._blob_callback(blob, self.callback_metadata())
def _filter_files(self, commit):
def filename_matches(path_expression, pathname):
''' Returns whether path_expression matches pathname or a leading
directory thereof, allowing path_expression to not have a trailing
slash even if it is meant to match a leading directory. '''
if path_expression == b'':
return True
n = len(path_expression)
if (pathname.startswith(path_expression) and
(path_expression[n-1:n] == b'/' or
len(pathname) == n or
pathname[n:n+1] == b'/')):
return True
return False
def newname(path_changes, pathname, use_base_name, filtering_is_inclusive):
''' Applies filtering and rename changes from path_changes to pathname,
returning any of None (file isn't wanted), original filename (file
is wanted with original name), or new filename. '''
wanted = False
full_pathname = pathname
if use_base_name:
pathname = os.path.basename(pathname)
for (mod_type, match_type, path_exp) in path_changes:
if mod_type == 'filter' and not wanted:
assert match_type in ('match', 'glob', 'regex')
if match_type == 'match' and filename_matches(path_exp, pathname):
wanted = True
if match_type == 'glob' and fnmatch.fnmatch(pathname, path_exp):
wanted = True
if match_type == 'regex' and path_exp.search(pathname):
wanted = True
elif mod_type == 'rename':
match, repl = path_exp
assert match_type in ('match','regex') # glob was translated to regex
if match_type == 'match' and filename_matches(match, full_pathname):
full_pathname = full_pathname.replace(match, repl, 1)
if match_type == 'regex':
full_pathname = match.sub(repl, full_pathname)
return full_pathname if (wanted == filtering_is_inclusive) else None
args = self._args
new_file_changes = {} # Assumes no renames or copies, otherwise collisions
for change in commit.file_changes:
# NEEDSWORK: _If_ we ever want to pass `--full-tree` to fast-export and
# parse that output, we'll need to modify this block; `--full-tree`
# issues a deleteall directive which has no filename, and thus this
# block would normally strip it. Of course, FileChange() and
# _parse_optional_filechange() would need updates too.
if change.type == b'DELETEALL':
new_file_changes[b''] = change
continue
if change.filename in self._newnames:
change.filename = self._newnames[change.filename]
else:
original_filename = change.filename
change.filename = newname(args.path_changes, change.filename,
args.use_base_name, args.inclusive)
if self._filename_callback:
change.filename = self._filename_callback(change.filename)
self._newnames[original_filename] = change.filename
if not change.filename:
continue # Filtering criteria excluded this file; move on to next one
if change.filename in new_file_changes:
# Getting here means that path renaming is in effect, and caused one
# path to collide with another. That's usually bad, but can be okay
# under two circumstances:
# 1) Sometimes people have a file named OLDFILE in old revisions of
# history, and they rename to NEWFILE, and would like to rewrite
# history so that all revisions refer to it as NEWFILE. As such,
# we can allow a collision when (at least) one of the two paths
# is a deletion. Note that if OLDFILE and NEWFILE are unrelated
# this also allows the rewrite to continue, which makes sense
# since OLDFILE is no longer in the way.
# 2) If OLDFILE and NEWFILE are exactly equal, then writing them
# both to the same location poses no problem; we only need one
# file. (This could come up if someone copied a file in some
# commit, then later either deleted the file or kept it exactly
# in sync with the original with any changes, and then decides
# they want to rewrite history to only have one of the two files)
colliding_change = new_file_changes[change.filename]
if change.type == b'D':
# We can just throw this one away and keep the other
continue
elif change.type == b'M' and (
change.mode == colliding_change.mode and
change.blob_id == colliding_change.blob_id):
# The two are identical, so we can throw this one away and keep other
continue
elif new_file_changes[change.filename].type != b'D':
raise SystemExit(_("File renaming caused colliding pathnames!\n") +
_(" Commit: {}\n").format(commit.original_id) +
_(" Filename: {}").format(change.filename))
# Strip files that are too large
if self._args.max_blob_size and \
self._unpacked_size.get(change.blob_id, 0) > self._args.max_blob_size:
continue
if self._args.strip_blobs_with_ids and \
change.blob_id in self._args.strip_blobs_with_ids:
continue
# Otherwise, record the change
new_file_changes[change.filename] = change
commit.file_changes = [v for k,v in sorted(new_file_changes.items())]
def _tweak_commit(self, commit, aux_info):
# Change the commit message according to callback
if not self._args.preserve_commit_hashes:
commit.message = self._hash_re.sub(self._translate_commit_hash,
commit.message)
if self._args.replace_message:
for literal, replacement in self._args.replace_message['literals']:
commit.message = commit.message.replace(literal, replacement)
for regex, replacement in self._args.replace_message['regexes']:
commit.message = regex.sub(replacement, commit.message)
if self._message_callback:
commit.message = self._message_callback(commit.message)
# Change the author & committer according to mailmap rules
args = self._args
if args.mailmap:
commit.author_name, commit.author_email = \
args.mailmap.translate(commit.author_name, commit.author_email)
commit.committer_name, commit.committer_email = \
args.mailmap.translate(commit.committer_name, commit.committer_email)
# Change author & committer according to callbacks
if self._name_callback:
commit.author_name = self._name_callback(commit.author_name)
commit.committer_name = self._name_callback(commit.committer_name)
if self._email_callback:
commit.author_email = self._email_callback(commit.author_email)
commit.committer_email = self._email_callback(commit.committer_email)
# Sometimes the 'branch' given is a tag; if so, rename it as requested so
# we don't get any old tagnames
if self._args.tag_rename:
commit.branch = RepoFilter._do_tag_rename(args.tag_rename, commit.branch)
if self._refname_callback:
commit.branch = self._refname_callback(commit.branch)
# Filter or rename the list of file changes
orig_file_changes = set(commit.file_changes)
self._filter_files(commit)
# Record ancestry graph
parents, orig_parents = commit.parents, aux_info['orig_parents']
if self._args.state_branch:
external_parents = parents
else:
external_parents = [p for p in parents if not isinstance(p, int)]
self._graph.record_external_commits(external_parents)
self._orig_graph.record_external_commits(external_parents)
self._graph.add_commit_and_parents(commit.id, parents)
self._orig_graph.add_commit_and_parents(commit.old_id, orig_parents)
# Prune parents (due to pruning of empty commits) if relevant
old_1st_parent = parents[0] if parents else None
parents, new_1st_parent = self._trim_extra_parents(orig_parents, parents)
commit.parents = parents
# If parents were pruned, then we need our file changes to be relative
# to the new first parent
if parents and old_1st_parent != parents[0]:
commit.file_changes = GitUtils.get_file_changes(self._repo_working_dir,
ID_TO_HASH[parents[0]],
commit.original_id)
orig_file_changes = set(commit.file_changes)
self._filter_files(commit)
# Find out which files were modified by the callbacks. Such paths could
# lead to subsequent commits being empty (e.g. if removing a line containing
# a password from every version of a file that had the password, and some
# later commit did nothing more than remove that line)
final_file_changes = set(commit.file_changes)
if self._args.replace_text or self._blob_callback:
differences = orig_file_changes.union(final_file_changes)
else:
differences = orig_file_changes.symmetric_difference(final_file_changes)
self._files_tweaked.update(x.filename for x in differences)
# Call the user-defined callback, if any
if self._commit_callback:
self._commit_callback(commit, self.callback_metadata(aux_info))
# Now print the resulting commit, or if prunable skip it
if not commit.dumped:
if not self._prunable(commit, new_1st_parent,
aux_info['had_file_changes'], orig_parents):
self._insert_into_stream(commit)
self._record_remapping(commit, orig_parents)
else:
rewrite_to = new_1st_parent or commit.first_parent()
commit.skip(new_id = rewrite_to)
if self._args.state_branch:
alias = Alias(commit.old_id or commit.id, rewrite_to or deleted_hash)
self._insert_into_stream(alias)
reset = Reset(commit.branch, rewrite_to or deleted_hash)
self._insert_into_stream(reset)
self._commit_renames[commit.original_id] = None
# Show progress
self._num_commits += 1
if not self._args.quiet:
self._progress_writer.show(self._parsed_message % self._num_commits)
@staticmethod
def _do_tag_rename(rename_pair, tagname):
old, new = rename_pair.split(b':', 1)
old, new = b'refs/tags/'+old, b'refs/tags/'+new
if tagname.startswith(old):
return tagname.replace(old, new, 1)
return tagname
def _tweak_tag(self, tag):
# Tweak the tag message according to callbacks
if self._args.replace_message:
for literal, replacement in self._args.replace_message['literals']:
tag.message = tag.message.replace(literal, replacement)
for regex, replacement in self._args.replace_message['regexes']:
tag.message = regex.sub(replacement, tag.message)
if self._message_callback:
tag.message = self._message_callback(tag.message)
# Tweak the tag name according to tag-name-related callbacks
tag_prefix = b'refs/tags/'
fullref = tag_prefix+tag.ref
if self._args.tag_rename:
fullref = RepoFilter._do_tag_rename(self._args.tag_rename, fullref)
if self._refname_callback:
fullref = self._refname_callback(fullref)
if not fullref.startswith(tag_prefix):
msg = "Error: fast-import requires tags to be in refs/tags/ namespace."
msg += "\n {} renamed to {}".format(tag_prefix+tag.ref, fullref)
raise SystemExit(msg)
tag.ref = fullref[len(tag_prefix):]
# Tweak the tagger according to callbacks
if self._args.mailmap:
tag.tagger_name, tag.tagger_email = \
self._args.mailmap.translate(tag.tagger_name, tag.tagger_email)
if self._name_callback:
tag.tagger_name = self._name_callback(tag.tagger_name)
if self._email_callback:
tag.tagger_email = self._email_callback(tag.tagger_email)
# Call general purpose tag callback
if self._tag_callback:
self._tag_callback(tag, self.callback_metadata())
def _tweak_reset(self, reset):
if self._args.tag_rename:
reset.ref = RepoFilter._do_tag_rename(self._args.tag_rename, reset.ref)
if self._refname_callback:
reset.ref = self._refname_callback(reset.ref)
if self._reset_callback:
self._reset_callback(reset, self.callback_metadata())
def results_tmp_dir(self, create_if_missing=True):
target_working_dir = self._args.target or b'.'
git_dir = GitUtils.determine_git_dir(target_working_dir)
d = os.path.join(git_dir, b'filter-repo')
if create_if_missing and not os.path.isdir(d):
os.mkdir(d)
return d
def _load_marks_file(self, marks_basename):
full_branch = 'refs/heads/{}'.format(self._args.state_branch)
marks_file = os.path.join(self.results_tmp_dir(), marks_basename)
working_dir = self._args.target or b'.'
cmd = ['git', '-C', working_dir, 'show-ref', full_branch]
contents = b''
if subproc.call(cmd, stdout=subprocess.DEVNULL) == 0:
cmd = ['git', '-C', working_dir, 'show',
'%s:%s' % (full_branch, decode(marks_basename))]
try:
contents = subproc.check_output(cmd)
except subprocess.CalledProcessError as e: # pragma: no cover
raise SystemExit(_("Failed loading %s from %s") %
(decode(marks_basename), full_branch))
if contents:
biggest_id = max(int(x.split()[0][1:]) for x in contents.splitlines())
_IDS._next_id = max(_IDS._next_id, biggest_id+1)
with open(marks_file, 'bw') as f:
f.write(contents)
return marks_file
def _save_marks_files(self):
basenames = [b'source-marks', b'target-marks']
working_dir = self._args.target or b'.'
# Check whether the branch exists
parent = []
full_branch = 'refs/heads/{}'.format(self._args.state_branch)
cmd = ['git', '-C', working_dir, 'show-ref', full_branch]
if subproc.call(cmd, stdout=subprocess.DEVNULL) == 0:
parent = ['-p', full_branch]
# Run 'git hash-object $MARKS_FILE' for each marks file, save result
blob_hashes = {}
for marks_basename in basenames:
marks_file = os.path.join(self.results_tmp_dir(), marks_basename)
if not os.path.isfile(marks_file): # pragma: no cover
raise SystemExit(_("Failed to find %s to save to %s")
% (marks_file, self._args.state_branch))
cmd = ['git', '-C', working_dir, 'hash-object', '-w', marks_file]
blob_hashes[marks_basename] = subproc.check_output(cmd).strip()
# Run 'git mktree' to create a tree out of it
p = subproc.Popen(['git', '-C', working_dir, 'mktree'],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
for b in basenames:
p.stdin.write(b'100644 blob %s\t%s\n' % (blob_hashes[b], b))
p.stdin.close()
p.wait()
tree = p.stdout.read().strip()
# Create the new commit
cmd = (['git', '-C', working_dir, 'commit-tree', '-m', 'New mark files',
tree] + parent)
commit = subproc.check_output(cmd).strip()
subproc.call(['git', '-C', working_dir, 'update-ref', full_branch, commit])
def importer_only(self):
self._run_sanity_checks()
self._setup_output()
def set_output(self, outputRepoFilter):
assert outputRepoFilter._output
# set_output implies this RepoFilter is doing exporting, though may not
# be the only one.
self._setup_input(use_done_feature = False)
# Set our output management up to pipe to outputRepoFilter's locations
self._managed_output = False
self._output = outputRepoFilter._output
self._import_pipes = outputRepoFilter._import_pipes
# Handle sanity checks, though currently none needed for export-only cases
self._run_sanity_checks()
def _setup_input(self, use_done_feature):
if self._args.stdin:
self._input = sys.stdin.detach()
sys.stdin = None # Make sure no one tries to accidentally use it
self._fe_orig = None
else:
skip_blobs = (self._blob_callback is None and
self._args.replace_text is None and
self._args.source == self._args.target)
extra_flags = []
if skip_blobs:
extra_flags.append('--no-data')
if self._args.max_blob_size:
self._unpacked_size, packed_size = GitUtils.get_blob_sizes()
if use_done_feature:
extra_flags.append('--use-done-feature')
if write_marks:
extra_flags.append(b'--mark-tags')
if self._args.state_branch:
assert(write_marks)
source_marks_file = self._load_marks_file(b'source-marks')
extra_flags.extend([b'--export-marks='+source_marks_file,
b'--import-marks='+source_marks_file])
if self._args.preserve_commit_encoding is not None: # pragma: no cover
reencode = 'no' if self._args.preserve_commit_encoding else 'yes'
extra_flags.append('--reencode='+reencode)
location = ['-C', self._args.source] if self._args.source else []
fep_cmd = ['git'] + location + ['fast-export', '--show-original-ids',
'--signed-tags=strip', '--tag-of-filtered-object=rewrite',
'--fake-missing-tagger', '--reference-excluded-parents'
] + extra_flags + self._args.refs
self._fep = subproc.Popen(fep_cmd, bufsize=-1, stdout=subprocess.PIPE)
self._input = self._fep.stdout
if self._args.dry_run or self._args.debug:
self._fe_orig = os.path.join(self.results_tmp_dir(),
b'fast-export.original')
output = open(self._fe_orig, 'bw')
self._input = InputFileBackup(self._input, output)
if self._args.debug:
tmp = [decode(x) if isinstance(x, bytes) else x for x in fep_cmd]
print("[DEBUG] Running: {}".format(' '.join(tmp)))
print(" (saving a copy of the output at {})"
.format(decode(self._fe_orig)))
def _setup_output(self):
if not self._args.dry_run:
location = ['-C', self._args.target] if self._args.target else []
fip_cmd = ['git'] + location + ['-c', 'core.ignorecase=false',
'fast-import', '--force', '--quiet']
if date_format_permissive:
fip_cmd.append('--date-format=raw-permissive')
if self._args.state_branch:
target_marks_file = self._load_marks_file(b'target-marks')
fip_cmd.extend([b'--export-marks='+target_marks_file,
b'--import-marks='+target_marks_file])
self._fip = subproc.Popen(fip_cmd, bufsize=-1,
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
self._import_pipes = (self._fip.stdin, self._fip.stdout)
if self._args.dry_run or self._args.debug:
self._fe_filt = os.path.join(self.results_tmp_dir(),
b'fast-export.filtered')
self._output = open(self._fe_filt, 'bw')
else:
self._output = self._fip.stdin
if self._args.debug and not self._args.dry_run:
self._output = DualFileWriter(self._fip.stdin, self._output)
tmp = [decode(x) if isinstance(x, bytes) else x for x in fip_cmd]
print("[DEBUG] Running: {}".format(' '.join(tmp)))
print(" (using the following file as input: {})"
.format(decode(self._fe_filt)))
def _migrate_origin_to_heads(self):
refs_to_migrate = set(x for x in self._orig_refs
if x.startswith(b'refs/remotes/origin/'))
if not refs_to_migrate:
return
if self._args.debug:
print("[DEBUG] Migrating refs/remotes/origin/* -> refs/heads/*")
target_working_dir = self._args.target or b'.'
p = subproc.Popen('git update-ref --no-deref --stdin'.split(),
stdin=subprocess.PIPE, cwd=target_working_dir)
for ref in refs_to_migrate:
if ref == b'refs/remotes/origin/HEAD':
p.stdin.write(b'delete %s %s\n' % (ref, self._orig_refs[ref]))
del self._orig_refs[ref]
continue
newref = ref.replace(b'refs/remotes/origin/', b'refs/heads/')
if newref not in self._orig_refs:
p.stdin.write(b'create %s %s\n' % (newref, self._orig_refs[ref]))
p.stdin.write(b'delete %s %s\n' % (ref, self._orig_refs[ref]))
self._orig_refs[newref] = self._orig_refs[ref]
del self._orig_refs[ref]
p.stdin.close()
if p.wait():
raise SystemExit(_("git update-ref failed; see above")) # pragma: no cover
# Now remove
if self._args.debug:
print("[DEBUG] Removing 'origin' remote (rewritten history will no ")
print(" longer be related; consider re-pushing it elsewhere.")
subproc.call('git remote rm origin'.split(), cwd=target_working_dir)
def _final_commands(self):
self._finalize_handled = True
self._done_callback and self._done_callback()
if not self._args.quiet:
self._progress_writer.finish()
def _ref_update(self, target_working_dir):
# Start the update-ref process
p = subproc.Popen('git update-ref --no-deref --stdin'.split(),
stdin=subprocess.PIPE,
cwd=target_working_dir)
# Remove replace_refs from _orig_refs
replace_refs = {k:v for k, v in self._orig_refs.items()
if k.startswith(b'refs/replace/')}
reverse_replace_refs = collections.defaultdict(list)
for k,v in replace_refs.items():
reverse_replace_refs[v].append(k)
all(map(self._orig_refs.pop, replace_refs))
# Remove unused refs
exported_refs, imported_refs = self.get_exported_and_imported_refs()
refs_to_nuke = exported_refs - imported_refs
if self._args.partial:
refs_to_nuke = set()
if refs_to_nuke and self._args.debug:
print("[DEBUG] Deleting the following refs:\n "+
decode(b"\n ".join(refs_to_nuke)))
p.stdin.write(b''.join([b"delete %s\n" % x
for x in refs_to_nuke]))
# Delete or update and add replace_refs; note that fast-export automatically
# handles 'update-no-add', we only need to take action for the other four
# choices for replace_refs.
self._flush_renames()
actual_renames = {k:v for k,v in self._commit_renames.items() if k != v}
if self._args.replace_refs in ['delete-no-add', 'delete-and-add']:
# Delete old replace refs, if unwanted
replace_refs_to_nuke = set(replace_refs)
if self._args.replace_refs == 'delete-and-add':
# git-update-ref won't allow us to update a ref twice, so be careful
# to avoid deleting refs we'll later update
replace_refs_to_nuke = replace_refs_to_nuke.difference(
[b'refs/replace/'+x for x in actual_renames])
p.stdin.write(b''.join([b"delete %s\n" % x
for x in replace_refs_to_nuke]))
if self._args.replace_refs in ['delete-and-add', 'update-or-add',
'update-and-add']:
# Add new replace refs
update_only = (self._args.replace_refs == 'update-or-add')
p.stdin.write(b''.join([b"update refs/replace/%s %s\n" % (old, new)
for old,new in actual_renames.items()
if new and not (update_only and
old in reverse_replace_refs)]))
# Complete the update-ref process
p.stdin.close()
if p.wait():
raise SystemExit(_("git update-ref failed; see above")) # pragma: no cover
def _record_metadata(self, metadata_dir, orig_refs):
self._flush_renames()
with open(os.path.join(metadata_dir, b'commit-map'), 'bw') as f:
f.write(("%-40s %s\n" % (_("old"), _("new"))).encode())
for (old,new) in self._commit_renames.items():
msg = b'%s %s\n' % (old, new if new != None else deleted_hash)
f.write(msg)
exported_refs, imported_refs = self.get_exported_and_imported_refs()
batch_check_process = None
batch_check_output_re = re.compile(b'^([0-9a-f]{40}) ([a-z]+) ([0-9]+)$')
with open(os.path.join(metadata_dir, b'ref-map'), 'bw') as f:
f.write(("%-40s %-40s %s\n" % (_("old"), _("new"), _("ref"))).encode())
for refname, old_hash in orig_refs.items():
if refname not in exported_refs:
continue
if refname not in imported_refs:
new_hash = deleted_hash
elif old_hash in self._commit_renames:
new_hash = self._commit_renames[old_hash]
new_hash = new_hash if new_hash != None else deleted_hash
else: # Must be either an annotated tag, or a ref whose tip was pruned
if not batch_check_process:
cmd = 'git cat-file --batch-check'.split()
target_working_dir = self._args.target or b'.'
batch_check_process = subproc.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
cwd=target_working_dir)
batch_check_process.stdin.write(refname+b"\n")
batch_check_process.stdin.flush()
line = batch_check_process.stdout.readline()
m = batch_check_output_re.match(line)
if m and m.group(2) in (b'tag', b'commit'):
new_hash = m.group(1)
elif line.endswith(b' missing\n'):
new_hash = deleted_hash
else:
raise SystemExit(_("Failed to find new id for %(refname)s "
"(old id was %(old_hash)s)")
% ({'refname': refname, 'old_hash': old_hash})
) # pragma: no cover
f.write(b'%s %s %s\n' % (old_hash, new_hash, refname))
if self._args.source or self._args.target:
new_refs = GitUtils.get_refs(self._args.target or b'.')
for ref, new_hash in new_refs.items():
if ref not in orig_refs and not ref.startswith(b'refs/replace/'):
old_hash = b'0'*len(new_hash)
f.write(b'%s %s %s\n' % (old_hash, new_hash, ref))
if batch_check_process:
batch_check_process.stdin.close()
batch_check_process.wait()
with open(os.path.join(metadata_dir, b'suboptimal-issues'), 'bw') as f:
issues_found = False
if self._commits_no_longer_merges:
issues_found = True
f.write(textwrap.dedent(_('''
The following commits used to be merge commits but due to filtering
are now regular commits; they likely have suboptimal commit messages
(e.g. "Merge branch next into master"). Original commit hash on the
left, commit hash after filtering/rewriting on the right:
''')[1:]).encode())
for oldhash, newhash in self._commits_no_longer_merges:
f.write(' {} {}\n'.format(oldhash, newhash).encode())
f.write(b'\n')
if self._commits_referenced_but_removed:
issues_found = True
f.write(textwrap.dedent(_('''
The following commits were filtered out, but referenced in another
commit message. The reference to the now-nonexistent commit hash
(or a substring thereof) was left as-is in any commit messages:
''')[1:]).encode())
for bad_commit_reference in self._commits_referenced_but_removed:
f.write(' {}\n'.format(bad_commit_reference).encode())
f.write(b'\n')
if not issues_found:
f.write(_("No filtering problems encountered.\n").encode())
with open(os.path.join(metadata_dir, b'already_ran'), 'bw') as f:
f.write(_("This file exists to allow you to filter again without --force.\n").encode())
def finish(self):
''' Alternative to run() when there is no input of our own to parse,
meaning that run only really needs to close the handle to fast-import
and let it finish, thus making a call to "run" feel like a misnomer. '''
assert not self._input
assert self._managed_output
self.run()
def insert(self, obj, direct_insertion = False):
if not direct_insertion:
if type(obj) == Blob:
self._tweak_blob(obj)
elif type(obj) == Commit:
aux_info = {'orig_parents': obj.parents,
'had_file_changes': bool(obj.file_changes)}
self._tweak_commit(obj, aux_info)
elif type(obj) == Reset:
self._tweak_reset(obj)
elif type(obj) == Tag:
self._tweak_tag(obj)
self._insert_into_stream(obj)
def _insert_into_stream(self, obj):
if not obj.dumped:
if self._parser:
self._parser.insert(obj)
else:
obj.dump(self._output)
def get_exported_and_imported_refs(self):
return self._parser.get_exported_and_imported_refs()
def run(self):
start = time.time()
if not self._input and not self._output:
self._run_sanity_checks()
if not self._args.dry_run and not self._args.partial:
self._migrate_origin_to_heads()
self._setup_input(use_done_feature = True)
self._setup_output()
assert self._sanity_checks_handled
if self._input:
# Create and run the filter
self._repo_working_dir = self._args.source or b'.'
self._parser = FastExportParser(blob_callback = self._tweak_blob,
commit_callback = self._tweak_commit,
tag_callback = self._tweak_tag,
reset_callback = self._tweak_reset,
done_callback = self._final_commands)
self._parser.run(self._input, self._output)
if not self._finalize_handled:
self._final_commands()
# Make sure fast-export completed successfully
if not self._args.stdin and self._fep.wait():
raise SystemExit(_("Error: fast-export failed; see above.")) # pragma: no cover
self._input.close()
# If we're not the manager of self._output, we should avoid post-run cleanup
if not self._managed_output:
return
# Close the output and ensure fast-import successfully completes
self._output.close()
if not self._args.dry_run and self._fip.wait():
raise SystemExit(_("Error: fast-import failed; see above.")) # pragma: no cover
# With fast-export and fast-import complete, update state if requested
if self._args.state_branch:
self._save_marks_files()
# Notify user how long it took, before doing a gc and such
msg = "New history written in {:.2f} seconds..."
if self._args.repack:
msg = "New history written in {:.2f} seconds; now repacking/cleaning..."
print(msg.format(time.time()-start))
# Exit early, if requested
if self._args.dry_run:
print(_("NOTE: Not running fast-import or cleaning up; --dry-run passed."))
if self._fe_orig:
print(_(" Requested filtering can be seen by comparing:"))
print(" " + decode(self._fe_orig))
else:
print(_(" Requested filtering can be seen at:"))
print(" " + decode(self._fe_filt))
return
target_working_dir = self._args.target or b'.'
if self._input:
self._ref_update(target_working_dir)
# Write out data about run
self._record_metadata(self.results_tmp_dir(), self._orig_refs)
# Final cleanup:
# If we need a repack, then nuke the reflogs and repack.
# If we need a reset, do a reset --hard
reset = not GitUtils.is_repository_bare(target_working_dir)
RepoFilter.cleanup(target_working_dir, self._args.repack, reset,
run_quietly=self._args.quiet,
show_debuginfo=self._args.debug)
# Let user know how long it took
print(_("Completely finished after {:.2f} seconds.")
.format(time.time()-start))
def main():
setup_gettext()
args = FilteringOptions.parse_args(sys.argv[1:])
if args.analyze:
RepoAnalyze.run(args)
else:
filter = RepoFilter(args)
filter.run()
if __name__ == '__main__':
main()
Reference in a new issue View git blame Copy permalink