# Copyright 2009-2010 Gregory P. Ward
# Copyright 2009-2010 Intelerad Medical Systems Incorporated
# Copyright 2010-2011 Fog Creek Software
# Copyright 2010-2011 Unity Technologies
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
'''High-level command function for lfconvert, plus the cmdtable.'''
import os, errno
import shutil
from mercurial import util, match as match_, hg, node, context, error, \
cmdutil, scmutil, commands
from mercurial.i18n import _
from mercurial.lock import release
import lfutil
import basestore
# -- Commands ----------------------------------------------------------
def lfconvert(ui, src, dest, *pats, **opts):
'''convert a normal repository to a largefiles repository
Convert repository SOURCE to a new repository DEST, identical to
SOURCE except that certain files will be converted as largefiles:
specifically, any file that matches any PATTERN *or* whose size is
above the minimum size threshold is converted as a largefile. The
size used to determine whether or not to track a file as a
largefile is the size of the first version of the file. The
minimum size can be specified either with --size or in
configuration as ``largefiles.size``.
After running this command you will need to make sure that
largefiles is enabled anywhere you intend to push the new
repository.
Use --to-normal to convert largefiles back to normal files; after
this, the DEST repository can be used without largefiles at all.'''
if opts['to_normal']:
tolfile = False
else:
tolfile = True
size = lfutil.getminsize(ui, True, opts.get('size'), default=None)
if not hg.islocal(src):
raise util.Abort(_('%s is not a local Mercurial repo') % src)
if not hg.islocal(dest):
raise util.Abort(_('%s is not a local Mercurial repo') % dest)
rsrc = hg.repository(ui, src)
ui.status(_('initializing destination %s\n') % dest)
rdst = hg.repository(ui, dest, create=True)
success = False
dstwlock = dstlock = None
try:
# Lock destination to prevent modification while it is converted to.
# Don't need to lock src because we are just reading from its history
# which can't change.
dstwlock = rdst.wlock()
dstlock = rdst.lock()
# Get a list of all changesets in the source. The easy way to do this
# is to simply walk the changelog, using changelog.nodesbetween().
# Take a look at mercurial/revlog.py:639 for more details.
# Use a generator instead of a list to decrease memory usage
ctxs = (rsrc[ctx] for ctx in rsrc.changelog.nodesbetween(None,
rsrc.heads())[0])
revmap = {node.nullid: node.nullid}
if tolfile:
lfiles = set()
normalfiles = set()
if not pats:
pats = ui.configlist(lfutil.longname, 'patterns', default=[])
if pats:
matcher = match_.match(rsrc.root, '', list(pats))
else:
matcher = None
lfiletohash = {}
for ctx in ctxs:
ui.progress(_('converting revisions'), ctx.rev(),
unit=_('revision'), total=rsrc['tip'].rev())
_lfconvert_addchangeset(rsrc, rdst, ctx, revmap,
lfiles, normalfiles, matcher, size, lfiletohash)
ui.progress(_('converting revisions'), None)
if os.path.exists(rdst.wjoin(lfutil.shortname)):
shutil.rmtree(rdst.wjoin(lfutil.shortname))
for f in lfiletohash.keys():
if os.path.isfile(rdst.wjoin(f)):
os.unlink(rdst.wjoin(f))
try:
os.removedirs(os.path.dirname(rdst.wjoin(f)))
except OSError:
pass
# If there were any files converted to largefiles, add largefiles
# to the destination repository's requirements.
if lfiles:
rdst.requirements.add('largefiles')
rdst._writerequirements()
else:
for ctx in ctxs:
ui.progress(_('converting revisions'), ctx.rev(),
unit=_('revision'), total=rsrc['tip'].rev())
_addchangeset(ui, rsrc, rdst, ctx, revmap)
ui.progress(_('converting revisions'), None)
success = True
finally:
rdst.dirstate.clear()
release(dstlock, dstwlock)
if not success:
# we failed, remove the new directory
shutil.rmtree(rdst.root)
def _addchangeset(ui, rsrc, rdst, ctx, revmap):
# Convert src parents to dst parents
parents = _convertparents(ctx, revmap)
# Generate list of changed files
files = _getchangedfiles(ctx, parents)
def getfilectx(repo, memctx, f):
if lfutil.standin(f) in files:
# if the file isn't in the manifest then it was removed
# or renamed, raise IOError to indicate this
try:
fctx = ctx.filectx(lfutil.standin(f))
except error.LookupError:
raise IOError
renamed = fctx.renamed()
if renamed:
renamed = lfutil.splitstandin(renamed[0])
hash = fctx.data().strip()
path = lfutil.findfile(rsrc, hash)
# If one file is missing, likely all files from this rev are
if path is None:
cachelfiles(ui, rsrc, ctx.node())
path = lfutil.findfile(rsrc, hash)
if path is None:
raise util.Abort(
_("missing largefile \'%s\' from revision %s")
% (f, node.hex(ctx.node())))
data = ''
fd = None
try:
fd = open(path, 'rb')
data = fd.read()
finally:
if fd:
fd.close()
return context.memfilectx(f, data, 'l' in fctx.flags(),
'x' in fctx.flags(), renamed)
else:
return _getnormalcontext(repo.ui, ctx, f, revmap)
dstfiles = []
for file in files:
if lfutil.isstandin(file):
dstfiles.append(lfutil.splitstandin(file))
else:
dstfiles.append(file)
# Commit
_commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap)
def _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, lfiles, normalfiles,
matcher, size, lfiletohash):
# Convert src parents to dst parents
parents = _convertparents(ctx, revmap)
# Generate list of changed files
files = _getchangedfiles(ctx, parents)
dstfiles = []
for f in files:
if f not in lfiles and f not in normalfiles:
islfile = _islfile(f, ctx, matcher, size)
# If this file was renamed or copied then copy
# the largefile-ness of its predecessor
if f in ctx.manifest():
fctx = ctx.filectx(f)
renamed = fctx.renamed()
renamedlfile = renamed and renamed[0] in lfiles
islfile |= renamedlfile
if 'l' in fctx.flags():
if renamedlfile:
raise util.Abort(
_('renamed/copied largefile %s becomes symlink')
% f)
islfile = False
if islfile:
lfiles.add(f)
else:
normalfiles.add(f)
if f in lfiles:
dstfiles.append(lfutil.standin(f))
# largefile in manifest if it has not been removed/renamed
if f in ctx.manifest():
fctx = ctx.filectx(f)
if 'l' in fctx.flags():
renamed = fctx.renamed()
if renamed and renamed[0] in lfiles:
raise util.Abort(_('largefile %s becomes symlink') % f)
# largefile was modified, update standins
m = util.sha1('')
m.update(ctx[f].data())
hash = m.hexdigest()
if f not in lfiletohash or lfiletohash[f] != hash:
rdst.wwrite(f, ctx[f].data(), ctx[f].flags())
executable = 'x' in ctx[f].flags()
lfutil.writestandin(rdst, lfutil.standin(f), hash,
executable)
lfiletohash[f] = hash
else:
# normal file
dstfiles.append(f)
def getfilectx(repo, memctx, f):
if lfutil.isstandin(f):
# if the file isn't in the manifest then it was removed
# or renamed, raise IOError to indicate this
srcfname = lfutil.splitstandin(f)
try:
fctx = ctx.filectx(srcfname)
except error.LookupError:
raise IOError
renamed = fctx.renamed()
if renamed:
# standin is always a largefile because largefile-ness
# doesn't change after rename or copy
renamed = lfutil.standin(renamed[0])
return context.memfilectx(f, lfiletohash[srcfname] + '\n', 'l' in
fctx.flags(), 'x' in fctx.flags(), renamed)
else:
return _getnormalcontext(repo.ui, ctx, f, revmap)
# Commit
_commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap)
def _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap):
mctx = context.memctx(rdst, parents, ctx.description(), dstfiles,
getfilectx, ctx.user(), ctx.date(), ctx.extra())
ret = rdst.commitctx(mctx)
rdst.setparents(ret)
revmap[ctx.node()] = rdst.changelog.tip()
# Generate list of changed files
def _getchangedfiles(ctx, parents):
files = set(ctx.files())
if node.nullid not in parents:
mc = ctx.manifest()
mp1 = ctx.parents()[0].manifest()
mp2 = ctx.parents()[1].manifest()
files |= (set(mp1) | set(mp2)) - set(mc)
for f in mc:
if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None):
files.add(f)
return files
# Convert src parents to dst parents
def _convertparents(ctx, revmap):
parents = []
for p in ctx.parents():
parents.append(revmap[p.node()])
while len(parents) < 2:
parents.append(node.nullid)
return parents
# Get memfilectx for a normal file
def _getnormalcontext(ui, ctx, f, revmap):
try:
fctx = ctx.filectx(f)
except error.LookupError:
raise IOError
renamed = fctx.renamed()
if renamed:
renamed = renamed[0]
data = fctx.data()
if f == '.hgtags':
data = _converttags (ui, revmap, data)
return context.memfilectx(f, data, 'l' in fctx.flags(),
'x' in fctx.flags(), renamed)
# Remap tag data using a revision map
def _converttags(ui, revmap, data):
newdata = []
for line in data.splitlines():
try:
id, name = line.split(' ', 1)
except ValueError:
ui.warn(_('skipping incorrectly formatted tag %s\n'
% line))
continue
try:
newid = node.bin(id)
except TypeError:
ui.warn(_('skipping incorrectly formatted id %s\n'
% id))
continue
try:
newdata.append('%s %s\n' % (node.hex(revmap[newid]),
name))
except KeyError:
ui.warn(_('no mapping for id %s\n') % id)
continue
return ''.join(newdata)
def _islfile(file, ctx, matcher, size):
'''Return true if file should be considered a largefile, i.e.
matcher matches it or it is larger than size.'''
# never store special .hg* files as largefiles
if file == '.hgtags' or file == '.hgignore' or file == '.hgsigs':
return False
if matcher and matcher(file):
return True
try:
return ctx.filectx(file).size() >= size * 1024 * 1024
except error.LookupError:
return False
def uploadlfiles(ui, rsrc, rdst, files):
'''upload largefiles to the central store'''
if not files:
return
store = basestore._openstore(rsrc, rdst, put=True)
at = 0
ui.debug("sending statlfile command for %d largefiles\n" % len(files))
retval = store.exists(files)
files = filter(lambda h: not retval[h], files)
ui.debug("%d largefiles need to be uploaded\n" % len(files))
for hash in files:
ui.progress(_('uploading largefiles'), at, unit='largefile',
total=len(files))
source = lfutil.findfile(rsrc, hash)
if not source:
raise util.Abort(_('largefile %s missing from store'
' (needs to be uploaded)') % hash)
# XXX check for errors here
store.put(source, hash)
at += 1
ui.progress(_('uploading largefiles'), None)
def verifylfiles(ui, repo, all=False, contents=False):
'''Verify that every largefile revision in the current changeset
exists in the central store. With --contents, also verify that
the contents of each local largefile file revision are correct (SHA-1 hash
matches the revision ID). With --all, check every changeset in
this repository.'''
if all:
# Pass a list to the function rather than an iterator because we know a
# list will work.
revs = range(len(repo))
else:
revs = ['.']
store = basestore._openstore(repo)
return store.verify(revs, contents=contents)
def debugdirstate(ui, repo):
'''Show basic information for the largefiles dirstate'''
lfdirstate = lfutil.openlfdirstate(ui, repo)
for file_, ent in sorted(lfdirstate._map.iteritems()):
mode = '%3o' % (ent[1] & 0777 & ~util.umask)
ui.write("%c %s %10d %s\n" % (ent[0], mode, ent[2], file_))
def cachelfiles(ui, repo, node, filelist=None):
'''cachelfiles ensures that all largefiles needed by the specified revision
are present in the repository's largefile cache.
returns a tuple (cached, missing). cached is the list of files downloaded
by this operation; missing is the list of files that were needed but could
not be found.'''
lfiles = lfutil.listlfiles(repo, node)
if filelist:
lfiles = set(lfiles) & set(filelist)
toget = []
for lfile in lfiles:
try:
expectedhash = repo[node][lfutil.standin(lfile)].data().strip()
except IOError, err:
if err.errno == errno.ENOENT:
continue # node must be None and standin wasn't found in wctx
raise
if not lfutil.findfile(repo, expectedhash):
toget.append((lfile, expectedhash))
if toget:
store = basestore._openstore(repo)
ret = store.get(toget)
return ret
return ([], [])
def downloadlfiles(ui, repo, rev=None):
matchfn = scmutil.match(repo[None],
[repo.wjoin(lfutil.shortname)], {})
def prepare(ctx, fns):
pass
totalsuccess = 0
totalmissing = 0
if rev != []: # walkchangerevs on empty list would return all revs
for ctx in cmdutil.walkchangerevs(repo, matchfn, {'rev' : rev},
prepare):
success, missing = cachelfiles(ui, repo, ctx.node())
totalsuccess += len(success)
totalmissing += len(missing)
ui.status(_("%d additional largefiles cached\n") % totalsuccess)
if totalmissing > 0:
ui.status(_("%d largefiles failed to download\n") % totalmissing)
return totalsuccess, totalmissing
def updatelfiles(ui, repo, filelist=None, printmessage=True):
wlock = repo.wlock()
try:
lfdirstate = lfutil.openlfdirstate(ui, repo)
lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate)
if filelist is not None:
lfiles = [f for f in lfiles if f in filelist]
printed = False
if printmessage and lfiles:
ui.status(_('getting changed largefiles\n'))
printed = True
cachelfiles(ui, repo, None, lfiles)
updated, removed = 0, 0
for f in lfiles:
i = _updatelfile(repo, lfdirstate, f)
if i:
if i > 0:
updated += i
else:
removed -= i
if printmessage and (removed or updated) and not printed:
ui.status(_('getting changed largefiles\n'))
printed = True
lfdirstate.write()
if printed and printmessage:
ui.status(_('%d largefiles updated, %d removed\n') % (updated,
removed))
finally:
wlock.release()
def _updatelfile(repo, lfdirstate, lfile):
'''updates a single largefile and copies the state of its standin from
the repository's dirstate to its state in the lfdirstate.
returns 1 if the file was modified, -1 if the file was removed, 0 if the
file was unchanged, and None if the needed largefile was missing from the
cache.'''
ret = 0
abslfile = repo.wjoin(lfile)
absstandin = repo.wjoin(lfutil.standin(lfile))
if os.path.exists(absstandin):
if os.path.exists(absstandin + '.orig') and os.path.exists(abslfile):
shutil.copyfile(abslfile, abslfile + '.orig')
expecthash = lfutil.readstandin(repo, lfile)
if (expecthash != '' and
(not os.path.exists(abslfile) or
expecthash != lfutil.hashfile(abslfile))):
if not lfutil.copyfromcache(repo, expecthash, lfile):
# use normallookup() to allocate entry in largefiles dirstate,
# because lack of it misleads lfilesrepo.status() into
# recognition that such cache missing files are REMOVED.
if lfile not in repo[None]: # not switched to normal file
util.unlinkpath(abslfile, ignoremissing=True)
lfdirstate.normallookup(lfile)
return None # don't try to set the mode
else:
# Synchronize largefile dirstate to the last modified time of
# the file
lfdirstate.normal(lfile)
ret = 1
mode = os.stat(absstandin).st_mode
if mode != os.stat(abslfile).st_mode:
os.chmod(abslfile, mode)
ret = 1
else:
# Remove lfiles for which the standin is deleted, unless the
# lfile is added to the repository again. This happens when a
# largefile is converted back to a normal file: the standin
# disappears, but a new (normal) file appears as the lfile.
if (os.path.exists(abslfile) and
repo.dirstate.normalize(lfile) not in repo[None]):
util.unlinkpath(abslfile)
ret = -1
state = repo.dirstate[lfutil.standin(lfile)]
if state == 'n':
# When rebasing, we need to synchronize the standin and the largefile,
# because otherwise the largefile will get reverted. But for commit's
# sake, we have to mark the file as unclean.
if getattr(repo, "_isrebasing", False):
lfdirstate.normallookup(lfile)
else:
lfdirstate.normal(lfile)
elif state == 'r':
lfdirstate.remove(lfile)
elif state == 'a':
lfdirstate.add(lfile)
elif state == '?':
lfdirstate.drop(lfile)
return ret
def lfpull(ui, repo, source="default", **opts):
"""pull largefiles for the specified revisions from the specified source
Pull largefiles that are referenced from local changesets but missing
locally, pulling from a remote repository to the local cache.
If SOURCE is omitted, the 'default' path will be used.
See :hg:`help urls` for more information.
.. container:: verbose
Some examples:
- pull largefiles for all branch heads::
hg lfpull -r "head() and not closed()"
- pull largefiles on the default branch::
hg lfpull -r "branch(default)"
"""
repo.lfpullsource = source
revs = opts.get('rev', [])
if not revs:
raise util.Abort(_('no revisions specified'))
revs = scmutil.revrange(repo, revs)
numcached = 0
for rev in revs:
ui.note(_('pulling largefiles for revision %s\n') % rev)
(cached, missing) = cachelfiles(ui, repo, rev)
numcached += len(cached)
ui.status(_("%d largefiles cached\n") % numcached)
# -- hg commands declarations ------------------------------------------------
cmdtable = {
'lfconvert': (lfconvert,
[('s', 'size', '',
_('minimum size (MB) for files to be converted '
'as largefiles'),
'SIZE'),
('', 'to-normal', False,
_('convert from a largefiles repo to a normal repo')),
],
_('hg lfconvert SOURCE DEST [FILE ...]')),
'lfpull': (lfpull,
[('r', 'rev', [], _('pull largefiles for these revisions'))
] + commands.remoteopts,
_('-r REV... [-e CMD] [--remotecmd CMD] [SOURCE]')
),
}
commands.inferrepo += " lfconvert"
|
